\documentclass{article}
\usepackage{graphicx}
\makeatletter
\newcommand*{\limitbox}[3]{%
  \begingroup
    \setlength{\@tempdima}{#1}%
    \setlength{\@tempdimb}{#2}%
    \resizebox{%
      \ifdim\width>\@tempdima\@tempdima\else\width\fi
    }{!}{%
      \resizebox{!}{%
        \ifdim\height>\@tempdimb\@tempdimb\else\height\fi
      }{#3}%
    }%
  \endgroup
}
\makeatother
\textwidth7cm% only for demo
\parindent0cm

\begin{document}
\rule{\textwidth}{1pt}
\begin{figure}
  \centering
  \limitbox{\linewidth}{\textheight}{\rule{2\textwidth}{3cm}}
  \caption{a very very  very very  very very  very very  very very very very  very very  
  very very  very very  very very  very very  very very  very very  very very  very very  very very  
  very very  very very very very  very very  very very  very very  very very  very very  
  very very  very very  very very  very very  very very  very very  very very 
  very very  very very  very very  very very  very very long caption}
\end{figure}

\begin{verbatim}
\usepackage{graphicx}

\makeatletter
\newcommand*{\limitbox}[3]{%
  \begingroup
    \setlength{\@tempdima}{#1}%
    \setlength{\@tempdimb}{#2}%
    \resizebox{%
      \ifdim\width>\@tempdima\@tempdima\else\width\fi
    }{!}{%
      \resizebox{!}{%
        \ifdim\height>\@tempdimb\@tempdimb\else\height\fi
      }{#3}%
    }%
  \endgroup
}
\makeatother
\end{verbatim}

\end{document}