13 #include <zypp-core/zyppng/base/EventDispatcher> 15 #include <zypp-core/zyppng/core/String> 18 #include <zypp-curl/CurlConfig> 19 #include <zypp-curl/auth/CurlAuthData> 20 #include <zypp-media/MediaConfig> 25 #include <curl/curl.h> 31 #include <boost/variant.hpp> 32 #include <boost/variant/polymorphic_get.hpp> 38 static size_t nwr_headerCallback (
char *ptr,
size_t size,
size_t nmemb,
void *userdata ) {
42 NetworkRequestPrivate *that =
reinterpret_cast<NetworkRequestPrivate *
>( userdata );
43 return that->headerfunction( ptr, size * nmemb );
45 static size_t nwr_writeCallback (
char *ptr,
size_t size,
size_t nmemb,
void *userdata ) {
49 NetworkRequestPrivate *that =
reinterpret_cast<NetworkRequestPrivate *
>( userdata );
50 return that->writefunction( ptr, {}, size * nmemb );
54 template<
class T>
struct always_false : std::false_type {};
58 : _outFile(
std::move(prevState._outFile) )
59 , _downloaded( prevState._downloaded )
60 , _partialHelper(
std::move(prevState._partialHelper) )
64 : _partialHelper(
std::move(prevState._partialHelper) )
68 : _outFile(
std::move(prevState._outFile) )
69 , _partialHelper(
std::move(prevState._partialHelper) )
70 , _downloaded( prevState._downloaded )
78 ,
_headers(
std::unique_ptr< curl_slist, decltype (&curl_slist_free_all) >(
nullptr, &curl_slist_free_all ) )
109 if ( urlScheme ==
"http" || urlScheme ==
"https" )
150 const auto &cHeaders =
_dispatcher->hostSpecificHeaders();
151 if (
auto i = cHeaders.find(
_url.
getHost()); i != cHeaders.end() ) {
152 for (
const auto &[key, value] : i->second ) {
154 "%s: %s", key.c_str(), value.c_str() )
165 case 4:
setCurlOption( CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4 );
break;
166 case 6:
setCurlOption( CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V6 );
break;
170 setCurlOption( CURLOPT_HEADERFUNCTION, &nwr_headerCallback );
185 if ( urlScheme ==
"https" )
206 #ifdef CURLSSLOPT_ALLOW_BEAST 208 setCurlOption( CURLOPT_SSL_OPTIONS, CURLSSLOPT_ALLOW_BEAST );
235 if (use_auth.empty())
236 use_auth =
"digest,basic";
238 if( auth != CURLAUTH_NONE)
240 DBG <<
_easyHandle <<
" " <<
"Enabling HTTP authentication methods: " << use_auth
241 <<
" (CURLOPT_HTTPAUTH=" << auth <<
")" << std::endl;
250 setCurlOption(CURLOPT_PROXYAUTH, CURLAUTH_BASIC|CURLAUTH_DIGEST|CURLAUTH_NTLM );
261 if ( proxyuserpwd.empty() )
266 DBG <<
_easyHandle <<
" " <<
"Proxy: ~/.curlrc does not contain the proxy-user option" << std::endl;
270 DBG <<
_easyHandle <<
" " <<
"Proxy: using proxy-user from ~/.curlrc" << std::endl;
278 if ( ! proxyuserpwd.empty() )
283 #if CURLVERSION_AT_LEAST(7,19,4) 288 DBG <<
_easyHandle <<
" " <<
"Proxy: explicitly NOPROXY" << std::endl;
303 #if CURLVERSION_AT_LEAST(7,15,5) 315 #if CURLVERSION_AT_LEAST(7,18,0) 322 for (
const auto &header : locSet.
headers() ) {
323 if ( !z_func()->addRequestHeader( header.c_str() ) )
335 return ( elem1.start < elem2.start );
339 if (
auto initState = std::get_if<pending_t>(&
_runningMode) ) {
342 initState->_partialHelper = std::make_unique<CurlMultiPartHandler>(
348 helper = initState->_partialHelper.get();
350 }
else if (
auto pendingState = std::get_if<prepareNextRangeBatch_t>(&
_runningMode) ) {
351 helper = pendingState->_partialHelper.get();
353 errBuf =
"Request is in invalid state to call setupHandle";
357 if ( !helper->prepare () ) {
358 errBuf = helper->lastErrorMessage ();
375 auto rmode = std::get_if<NetworkRequestPrivate::running_t>( &
_runningMode );
377 DBG <<
_easyHandle <<
"Can only create output file in running mode" << std::endl;
381 if ( !rmode->_outFile ) {
382 std::string openMode =
"w+b";
393 if ( !rmode->_outFile ) {
406 auto rmode = std::get_if<NetworkRequestPrivate::running_t>( &
_runningMode );
407 if ( rmode->_partialHelper )
return rmode->_partialHelper->canRecover();
413 auto rmode = std::get_if<NetworkRequestPrivate::running_t>( &
_runningMode );
415 errBuf =
"NetworkRequestPrivate::prepareToContinue called in invalid state";
419 if ( rmode->_partialHelper && rmode->_partialHelper->hasMoreWork() ) {
425 auto &prepMode = std::get<prepareNextRangeBatch_t>(
_runningMode);
426 if ( !prepMode._partialHelper->prepareToContinue() ) {
427 errBuf = prepMode._partialHelper->lastErrorMessage();
431 if ( hadRangeFail ) {
445 errBuf =
"Request has no more work";
458 bool isRangeContinuation = std::holds_alternative<prepareNextRangeBatch_t>(
_runningMode );
459 if ( isRangeContinuation ) {
460 MIL <<
_easyHandle <<
" " <<
"Continuing a previously started range batch." << std::endl;
468 if ( m._activityTimer ) {
471 m._activityTimer->start( static_cast<uint64_t>(
_settings.
timeout() * 1000 ) );
474 if ( !isRangeContinuation )
480 if ( std::holds_alternative<running_t>(
_runningMode) ) {
482 if ( rmode._partialHelper )
483 rmode._partialHelper->finalize();
490 resState.
_result = std::move(err);
492 if ( std::holds_alternative<running_t>(
_runningMode) ) {
501 if ( !rmode._partialHelper->verifyData() ){
508 if ( fseek( rmode._outFile, 0, SEEK_SET ) != 0 ) {
511 constexpr
size_t bufSize = 4096;
513 while(
auto cnt = fread(buf, 1, bufSize, rmode._outFile ) > 0 ) {
525 if ( calcSum != expSum ) {
534 rmode._outFile.reset();
559 std::map<std::string, boost::any> extraInfo;
560 extraInfo.insert( {
"requestUrl",
_url } );
572 if ( std::holds_alternative<running_t>(
_runningMode ) ){
574 if ( rmode._activityTimer && rmode._activityTimer->isRunning() )
575 rmode._activityTimer->start();
585 if ( !std::holds_alternative<running_t>(that->
_runningMode) ){
586 DBG << that->
_easyHandle <<
" " <<
"Curl progress callback was called in invalid state "<< that->z_func()->state() << std::endl;
590 auto &rmode = std::get<running_t>( that->
_runningMode );
595 rmode._isInCallback =
true;
596 if ( rmode._lastProgressNow != dlnow ) {
597 rmode._lastProgressNow = dlnow;
598 that->
_sigProgress.emit( *that->z_func(), dltotal, dlnow, ultotal, ulnow );
600 rmode._isInCallback =
false;
602 return rmode._cachedResult ? CURLE_ABORTED_BY_CALLBACK : CURLE_OK;
615 std::string_view hdr( ptr, bytes );
617 hdr.remove_prefix( std::min( hdr.find_first_not_of(
" \t\r\n"), hdr.size() ) );
618 const auto lastNonWhitespace = hdr.find_last_not_of(
" \t\r\n");
619 if ( lastNonWhitespace != hdr.npos )
620 hdr.remove_suffix( hdr.size() - (lastNonWhitespace + 1) );
622 hdr = std::string_view();
629 const auto &repSize = rmode._partialHelper->reportedFileSize ();
638 (void)curl_easy_getinfo(
_easyHandle, CURLINFO_RESPONSE_CODE, &statuscode);
650 auto str = std::string ( lenStr.data(), lenStr.length() );
651 auto len = zypp::str::strtonum<typename zypp::ByteCount::SizeType>(
str.data() );
653 DBG <<
_easyHandle <<
" " <<
"Got Content-Length Header: " << len << std::endl;
683 if ( fseek( rmode._outFile, *offset, SEEK_SET ) != 0 ) {
685 "Unable to set output file pointer." );
688 rmode._currentFileOffset = *offset;
692 const auto &repSize = rmode._partialHelper->reportedFileSize ();
705 auto written = fwrite( data, 1, max, rmode._outFile );
714 rmode._currentFileOffset += written;
717 rmode._downloaded += written;
725 auto rmode = std::get_if<NetworkRequestPrivate::running_t>( &
_runningMode );
726 if ( !rmode || !rmode->_partialHelper || !rmode->_partialHelper->hasError() )
730 if ( rmode->_cachedResult )
734 MIL_MEDIA <<
_easyHandle <<
" Multipart handler announced error code " << lastErr.toString () << std::endl;
735 rmode->_cachedResult = lastErr;
749 if ( d->_dispatcher )
750 d->_dispatcher->cancel( *
this,
"Request destroyed while still running" );
755 d_func()->_expectedFileSize = std::move( expectedFileSize );
762 if (
state() ==
Pending && triggerReschedule && d->_dispatcher )
763 d->_dispatcher->reschedule();
768 return d_func()->_priority;
773 d_func()->_options = opt;
778 return d_func()->_options;
787 d->_requestedRanges.push_back(
Range::make( start, len, std::move(digest), std::move( expectedChkSum ), std::move( userData ), digestCompareLen, chksumpad ) );
796 d->_requestedRanges.push_back( std::move(range) );
797 auto &rng = d->_requestedRanges.back();
799 rng.bytesWritten = 0;
801 rng._digest->reset();
826 d->_requestedRanges.clear();
831 const auto mystate =
state();
837 std::vector<Range> failed;
838 for (
auto &r : d->_requestedRanges ) {
840 failed.push_back( r.clone() );
848 return d_func()->_requestedRanges;
853 return d_func()->_lastRedirect;
858 return d_func()->_easyHandle;
863 const auto myerr =
error();
864 const auto mystate =
state();
870 auto getMeasurement = [ this ](
const CURLINFO info, std::chrono::microseconds &target ){
871 using FPSeconds = std::chrono::duration<double, std::chrono::seconds::period>;
873 const auto res = curl_easy_getinfo( d_func()->_easyHandle, info, &val );
874 if ( CURLE_OK == res ) {
875 target = std::chrono::duration_cast<std::chrono::microseconds>( FPSeconds(val) );
879 getMeasurement( CURLINFO_NAMELOOKUP_TIME, t.
namelookup );
880 getMeasurement( CURLINFO_CONNECT_TIME, t.
connect);
881 getMeasurement( CURLINFO_APPCONNECT_TIME, t.
appconnect);
882 getMeasurement( CURLINFO_PRETRANSFER_TIME , t.
pretransfer);
883 getMeasurement( CURLINFO_TOTAL_TIME, t.
total);
884 getMeasurement( CURLINFO_REDIRECT_TIME, t.
redirect);
893 if ( !std::holds_alternative<NetworkRequestPrivate::running_t>( d->_runningMode) )
896 const auto &rmode = std::get<NetworkRequestPrivate::running_t>( d->_runningMode );
902 return d_func()->_url;
916 return d_func()->_targetFile;
924 d->_targetFile = path;
929 return d_func()->_fMode;
937 d->_fMode = std::move( mode );
943 if ( curl_easy_getinfo( d_func()->_easyHandle, CURLINFO_CONTENT_TYPE, &ptr ) == CURLE_OK && ptr )
944 return std::string(ptr);
945 return std::string();
951 using T = std::decay_t<decltype(arg)>;
952 if constexpr (std::is_same_v<T, NetworkRequestPrivate::pending_t> || std::is_same_v<T, NetworkRequestPrivate::prepareNextRangeBatch_t> )
954 else if constexpr (std::is_same_v<T, NetworkRequestPrivate::running_t>
955 || std::is_same_v<T, NetworkRequestPrivate::finished_t>)
956 return arg._contentLenght;
958 static_assert(always_false<T>::value,
"Unhandled state type");
959 }, d_func()->_runningMode);
965 using T = std::decay_t<decltype(arg)>;
966 if constexpr (std::is_same_v<T, NetworkRequestPrivate::pending_t>)
968 else if constexpr (std::is_same_v<T, NetworkRequestPrivate::running_t>
969 || std::is_same_v<T, NetworkRequestPrivate::prepareNextRangeBatch_t>
970 || std::is_same_v<T, NetworkRequestPrivate::finished_t>)
971 return arg._downloaded;
973 static_assert(always_false<T>::value,
"Unhandled state type");
974 }, d_func()->_runningMode);
979 return d_func()->_settings;
984 return std::visit([
this](
auto& arg) {
985 using T = std::decay_t<decltype(arg)>;
986 if constexpr (std::is_same_v<T, NetworkRequestPrivate::pending_t>)
988 else if constexpr (std::is_same_v<T, NetworkRequestPrivate::running_t> || std::is_same_v<T, NetworkRequestPrivate::prepareNextRangeBatch_t> )
990 else if constexpr (std::is_same_v<T, NetworkRequestPrivate::finished_t>) {
991 if ( std::get<NetworkRequestPrivate::finished_t>( d_func()->_runningMode ).
_result.isError() )
997 static_assert(always_false<T>::value,
"Unhandled state type");
998 }, d_func()->_runningMode);
1003 const auto s =
state();
1006 return std::get<NetworkRequestPrivate::finished_t>( d_func()->_runningMode).
_result;
1012 return std::string();
1026 curl_slist *res = curl_slist_append( d->_headers ? d->_headers.get() :
nullptr, header.c_str() );
1031 d->_headers = std::unique_ptr< curl_slist, decltype (&curl_slist_free_all) >( res, &curl_slist_free_all );
1038 return d_func()->_sigStarted;
1043 return d_func()->_sigBytesDownloaded;
1048 return d_func()->_sigProgress;
1053 return d_func()->_sigFinished;
std::string getScheme() const
Returns the scheme name of the URL.
Signal< void(NetworkRequest &req)> _sigStarted
std::string errorMessage() const
bool isError() const
isError Will return true if this is a actual error
T trim(StrType &&s, const Trim trim_r)
void setCurlOption(CURLoption opt, T data)
std::string curlUnEscape(const std::string &text_r)
std::optional< Timings > timings() const
After the request is finished query the timings that were collected during download.
void * nativeHandle() const
zypp::ByteCount reportedByteCount() const
Returns the number of bytes that are reported from the backend as the full download size...
const std::vector< Range > & requestedRanges() const
std::chrono::microseconds connect
std::array< char, CURL_ERROR_SIZE+1 > _errorBuf
uint64_t interval() const
std::optional< FileVerifyInfo > _fileVerification
The digest for the full file.
The CurlMultiPartHandler class.
#define ZYPP_THROW(EXCPT)
Drops a logline and throws the Exception.
void addRequestRange(size_t start, size_t len=0, std::optional< zypp::Digest > &&digest={}, CheckSumBytes expectedChkSum=CheckSumBytes(), std::any userData=std::any(), std::optional< size_t > digestCompareLen={}, std::optional< size_t > chksumpad={})
SignalProxy< void(NetworkRequest &req, zypp::ByteCount count)> sigBytesDownloaded()
Signals that new data has been downloaded, this is only the payload and does not include control data...
int assert_file_mode(const Pathname &path, unsigned mode)
Like assert_file but enforce mode even if the file already exists.
bool hasPrefixCI(const C_Str &str_r, const C_Str &prefix_r)
Compute Message Digests (MD5, SHA1 etc)
NetworkRequest::FileMode _fMode
Store and operate with byte count.
const std::string & lastRedirectInfo() const
const std::string _currentCookieFile
static Range make(size_t start, size_t len=0, std::optional< zypp::Digest > &&digest={}, CheckSumBytes &&expectedChkSum=CheckSumBytes(), std::any &&userData=std::any(), std::optional< size_t > digestCompareLen={}, std::optional< size_t > _dataBlockPadding={})
std::chrono::microseconds pretransfer
zypp::ByteCount downloadedByteCount() const
Returns the number of already downloaded bytes as reported by the backend.
NetworkRequest::Options _options
const char * c_str() const
String representation.
void resetActivityTimer()
String related utilities and Regular expression matching.
std::chrono::microseconds appconnect
constexpr bool always_false
running_t(pending_t &&prevState)
std::string nativeErrorString() const
Signal< void(NetworkRequest &req, zypp::ByteCount count)> _sigBytesDownloaded
void setOptions(Options opt)
std::string form(const char *format,...) __attribute__((format(printf
Printf style construction of std::string.
~NetworkRequestPrivate() override
void resetRequestRanges()
TransferSettings & transferSettings()
enum zyppng::NetworkRequestPrivate::ProtocolMode _protocolMode
void setExpectedFileSize(zypp::ByteCount expectedFileSize)
void setFileOpenMode(FileMode mode)
Sets the file open mode to mode.
bool hasError() const
Checks if there was a error with the request.
void onActivityTimeout(Timer &)
SignalProxy< void(Timer &t)> sigExpired()
This signal is always emitted when the timer expires.
int ZYPP_MEDIA_CURL_IPRESOLVE()
4/6 to force IPv4/v6
zypp::Pathname _targetFile
bool empty() const
Test for an empty path.
void setUrl(const Url &url)
This will change the URL of the request.
std::chrono::microseconds namelookup
std::string asString() const
Returns a default string representation of the Url object.
std::string getQueryParam(const std::string ¶m, EEncoding eflag=zypp::url::E_DECODED) const
Return the value for the specified query parameter.
bool addRequestHeader(const std::string &header)
std::string trim(const std::string &s, const Trim trim_r)
const std::string & asString() const
String representation.
Signal< void(NetworkRequest &req, off_t dltotal, off_t dlnow, off_t ultotal, off_t ulnow)> _sigProgress
std::string asString() const
Error message provided by dumpOn as string.
void notifyErrorCodeChanged() override
bool initialize(std::string &errBuf)
The NetworkRequestError class Represents a error that occured in.
The Timer class provides repetitive and single-shot timers.
std::vector< char > peekData(off_t offset, size_t count) const
NetworkRequestError error() const
Returns the last set Error.
zypp::ByteCount _expectedFileSize
std::string extendedErrorString() const
In some cases, curl can provide extended error information collected at runtime.
Priority priority() const
bool setupHandle(std::string &errBuf)
std::vector< char > peek_data_fd(FILE *fd, off_t offset, size_t count)
bool create(const std::string &name)
initialize creation of a new message digest
const zypp::Pathname & targetFilePath() const
Returns the target filename path.
size_t writefunction(char *ptr, std::optional< off_t > offset, size_t bytes) override
std::unique_ptr< curl_slist, decltype(&curl_slist_free_all) > _headers
uint64_t remaining() const
#define ZYPP_CAUGHT(EXCPT)
Drops a logline telling the Exception was caught (in order to handle it).
void setTargetFilePath(const zypp::Pathname &path)
Changes the target file path of the download.
static int curlProgressCallback(void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow)
std::string contentType() const
Returns the content type as reported from the server.
std::string strerr_cxx(const int err=-1)
static const Unit B
1 Byte
bool setExpectedFileChecksum(const zypp::CheckSum &expected)
Base class for Exception.
std::string _lastRedirect
to log/report redirections
std::chrono::microseconds total
bool any_of(const Container &c, Fnc &&cb)
std::string getHost(EEncoding eflag=zypp::url::E_DECODED) const
Returns the hostname or IP from the URL authority.
void setupZYPP_MEDIA_CURL_DEBUG(CURL *curl)
Setup CURLOPT_VERBOSE and CURLOPT_DEBUGFUNCTION according to env::ZYPP_MEDIA_CURL_DEBUG.
void setPriority(Priority prio, bool triggerReschedule=true)
State state() const
Returns the current state the HttpDownloadRequest is in.
TransferSettings _settings
NetworkRequestDispatcher * _dispatcher
bool strToBool(const C_Str &str, bool default_r)
Parse str into a bool depending on the default value.
ZYPP_IMPL_PRIVATE(UnixSignalSource)
std::vector< NetworkRequest::Range > _requestedRanges
the requested ranges that need to be downloaded
std::chrono::microseconds redirect
SignalProxy< void(NetworkRequest &req, const NetworkRequestError &err)> sigFinished()
Signals that the download finished.
Signal< void(NetworkRequest &req, const NetworkRequestError &err)> _sigFinished
size_t headerfunction(char *ptr, size_t bytes) override
Type type() const
type Returns the type of the error
SignalProxy< void(NetworkRequest &req)> sigStarted()
Signals that the dispatcher dequeued the request and actually starts downloading data.
zypp::ByteCount _contentLenght
#define EXPLICITLY_NO_PROXY
FileMode fileOpenMode() const
Returns the currently configured file open mode.
std::vector< Range > failedRanges() const
Easy-to use interface to the ZYPP dependency resolver.
NetworkRequestPrivate(Url &&url, zypp::Pathname &&targetFile, NetworkRequest::FileMode fMode, NetworkRequest &p)
CURLcode setCurlRedirProtocols(CURL *curl)
void setResult(NetworkRequestError &&err)
static zyppng::NetworkRequestError customError(NetworkRequestError::Type t, std::string &&errorMsg="", std::map< std::string, boost::any > &&extraInfo={})
SignalProxy< void(NetworkRequest &req, off_t dltotal, off_t dlnow, off_t ultotal, off_t ulnow)> sigProgress()
Signals if there was data read from the download.
bool prepareToContinue(std::string &errBuf)
~NetworkRequest() override
NetworkRequestError _result
ZYppCommitResult & _result
std::variant< pending_t, running_t, prepareNextRangeBatch_t, finished_t > _runningMode