First Commit
This commit is contained in:
321
common/HTTPDownloader.cpp
Normal file
321
common/HTTPDownloader.cpp
Normal file
@@ -0,0 +1,321 @@
|
||||
// SPDX-FileCopyrightText: 2002-2025 PCSX2 Dev Team
|
||||
// SPDX-License-Identifier: GPL-3.0+
|
||||
|
||||
#include "common/HTTPDownloader.h"
|
||||
#include "common/Assertions.h"
|
||||
#include "common/Console.h"
|
||||
#include "common/ProgressCallback.h"
|
||||
#include "common/StringUtil.h"
|
||||
#include "common/Timer.h"
|
||||
#include "common/Threading.h"
|
||||
|
||||
static constexpr float DEFAULT_TIMEOUT_IN_SECONDS = 30;
|
||||
static constexpr u32 DEFAULT_MAX_ACTIVE_REQUESTS = 4;
|
||||
|
||||
const char HTTPDownloader::DEFAULT_USER_AGENT[] =
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:85.0) Gecko/20100101 Firefox/85.0";
|
||||
|
||||
HTTPDownloader::HTTPDownloader()
|
||||
: m_timeout(DEFAULT_TIMEOUT_IN_SECONDS)
|
||||
, m_max_active_requests(DEFAULT_MAX_ACTIVE_REQUESTS)
|
||||
{
|
||||
}
|
||||
|
||||
HTTPDownloader::~HTTPDownloader() = default;
|
||||
|
||||
void HTTPDownloader::SetTimeout(float timeout)
|
||||
{
|
||||
m_timeout = timeout;
|
||||
}
|
||||
|
||||
void HTTPDownloader::SetMaxActiveRequests(u32 max_active_requests)
|
||||
{
|
||||
pxAssert(max_active_requests > 0);
|
||||
m_max_active_requests = max_active_requests;
|
||||
}
|
||||
|
||||
void HTTPDownloader::CreateRequest(std::string url, Request::Callback callback, ProgressCallback* progress)
|
||||
{
|
||||
Request* req = InternalCreateRequest();
|
||||
req->parent = this;
|
||||
req->type = Request::Type::Get;
|
||||
req->url = std::move(url);
|
||||
req->callback = std::move(callback);
|
||||
req->progress = progress;
|
||||
req->start_time = Common::Timer::GetCurrentValue();
|
||||
|
||||
std::unique_lock<std::mutex> lock(m_pending_http_request_lock);
|
||||
if (LockedGetActiveRequestCount() < m_max_active_requests)
|
||||
{
|
||||
if (!StartRequest(req))
|
||||
return;
|
||||
}
|
||||
|
||||
LockedAddRequest(req);
|
||||
}
|
||||
|
||||
void HTTPDownloader::CreatePostRequest(std::string url, std::string post_data, Request::Callback callback, ProgressCallback* progress)
|
||||
{
|
||||
Request* req = InternalCreateRequest();
|
||||
req->parent = this;
|
||||
req->type = Request::Type::Post;
|
||||
req->url = std::move(url);
|
||||
req->post_data = std::move(post_data);
|
||||
req->callback = std::move(callback);
|
||||
req->progress = progress;
|
||||
req->start_time = Common::Timer::GetCurrentValue();
|
||||
|
||||
std::unique_lock<std::mutex> lock(m_pending_http_request_lock);
|
||||
if (LockedGetActiveRequestCount() < m_max_active_requests)
|
||||
{
|
||||
if (!StartRequest(req))
|
||||
return;
|
||||
}
|
||||
|
||||
LockedAddRequest(req);
|
||||
}
|
||||
|
||||
void HTTPDownloader::LockedPollRequests(std::unique_lock<std::mutex>& lock)
|
||||
{
|
||||
if (m_pending_http_requests.empty())
|
||||
return;
|
||||
|
||||
InternalPollRequests();
|
||||
|
||||
const Common::Timer::Value current_time = Common::Timer::GetCurrentValue();
|
||||
u32 active_requests = 0;
|
||||
u32 unstarted_requests = 0;
|
||||
|
||||
for (size_t index = 0; index < m_pending_http_requests.size();)
|
||||
{
|
||||
Request* req = m_pending_http_requests[index];
|
||||
if (req->state == Request::State::Pending)
|
||||
{
|
||||
unstarted_requests++;
|
||||
index++;
|
||||
continue;
|
||||
}
|
||||
|
||||
if ((req->state == Request::State::Started || req->state == Request::State::Receiving) &&
|
||||
current_time >= req->start_time && Common::Timer::ConvertValueToSeconds(current_time - req->start_time) >= m_timeout)
|
||||
{
|
||||
// request timed out
|
||||
Console.Error("Request for '%s' timed out", req->url.c_str());
|
||||
|
||||
req->state.store(Request::State::Cancelled);
|
||||
m_pending_http_requests.erase(m_pending_http_requests.begin() + index);
|
||||
lock.unlock();
|
||||
|
||||
req->callback(HTTP_STATUS_TIMEOUT, std::string(), Request::Data());
|
||||
|
||||
CloseRequest(req);
|
||||
|
||||
lock.lock();
|
||||
continue;
|
||||
}
|
||||
else if ((req->state == Request::State::Started || req->state == Request::State::Receiving) && req->progress &&
|
||||
req->progress->IsCancelled())
|
||||
{
|
||||
// request timed out
|
||||
Console.Error("Request for '%s' cancelled", req->url.c_str());
|
||||
|
||||
req->state.store(Request::State::Cancelled);
|
||||
m_pending_http_requests.erase(m_pending_http_requests.begin() + index);
|
||||
lock.unlock();
|
||||
|
||||
req->callback(HTTP_STATUS_CANCELLED, std::string(), Request::Data());
|
||||
|
||||
CloseRequest(req);
|
||||
|
||||
lock.lock();
|
||||
continue;
|
||||
}
|
||||
|
||||
if (req->state != Request::State::Complete)
|
||||
{
|
||||
if (req->progress)
|
||||
{
|
||||
const u32 size = static_cast<u32>(req->data.size());
|
||||
if (size != req->last_progress_update)
|
||||
{
|
||||
req->last_progress_update = size;
|
||||
req->progress->SetProgressRange(req->content_length);
|
||||
req->progress->SetProgressValue(req->last_progress_update);
|
||||
}
|
||||
}
|
||||
|
||||
active_requests++;
|
||||
index++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// request complete
|
||||
DevCon.WriteLn("Request for '%s' complete, returned status code %u and %zu bytes", req->url.c_str(),
|
||||
req->status_code, req->data.size());
|
||||
m_pending_http_requests.erase(m_pending_http_requests.begin() + index);
|
||||
|
||||
// run callback with lock unheld
|
||||
lock.unlock();
|
||||
req->callback(req->status_code, req->content_type, std::move(req->data));
|
||||
CloseRequest(req);
|
||||
lock.lock();
|
||||
}
|
||||
|
||||
// start new requests when we finished some
|
||||
if (unstarted_requests > 0 && active_requests < m_max_active_requests)
|
||||
{
|
||||
for (size_t index = 0; index < m_pending_http_requests.size();)
|
||||
{
|
||||
Request* req = m_pending_http_requests[index];
|
||||
if (req->state != Request::State::Pending)
|
||||
{
|
||||
index++;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!StartRequest(req))
|
||||
{
|
||||
m_pending_http_requests.erase(m_pending_http_requests.begin() + index);
|
||||
continue;
|
||||
}
|
||||
|
||||
active_requests++;
|
||||
index++;
|
||||
|
||||
if (active_requests >= m_max_active_requests)
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void HTTPDownloader::PollRequests()
|
||||
{
|
||||
std::unique_lock<std::mutex> lock(m_pending_http_request_lock);
|
||||
LockedPollRequests(lock);
|
||||
}
|
||||
|
||||
void HTTPDownloader::WaitForAllRequests()
|
||||
{
|
||||
std::unique_lock<std::mutex> lock(m_pending_http_request_lock);
|
||||
while (!m_pending_http_requests.empty())
|
||||
{
|
||||
// Don't burn too much CPU.
|
||||
Threading::Sleep(1);
|
||||
LockedPollRequests(lock);
|
||||
}
|
||||
}
|
||||
|
||||
void HTTPDownloader::LockedAddRequest(Request* request)
|
||||
{
|
||||
m_pending_http_requests.push_back(request);
|
||||
}
|
||||
|
||||
u32 HTTPDownloader::LockedGetActiveRequestCount()
|
||||
{
|
||||
u32 count = 0;
|
||||
for (Request* req : m_pending_http_requests)
|
||||
{
|
||||
if (req->state == Request::State::Started || req->state == Request::State::Receiving)
|
||||
count++;
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
bool HTTPDownloader::HasAnyRequests()
|
||||
{
|
||||
std::unique_lock<std::mutex> lock(m_pending_http_request_lock);
|
||||
return !m_pending_http_requests.empty();
|
||||
}
|
||||
|
||||
std::string HTTPDownloader::GetExtensionForContentType(const std::string& content_type)
|
||||
{
|
||||
// Based on https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types
|
||||
static constexpr const char* table[][2] = {
|
||||
{"audio/aac", "aac"},
|
||||
{"application/x-abiword", "abw"},
|
||||
{"application/x-freearc", "arc"},
|
||||
{"image/avif", "avif"},
|
||||
{"video/x-msvideo", "avi"},
|
||||
{"application/vnd.amazon.ebook", "azw"},
|
||||
{"application/octet-stream", "bin"},
|
||||
{"image/bmp", "bmp"},
|
||||
{"application/x-bzip", "bz"},
|
||||
{"application/x-bzip2", "bz2"},
|
||||
{"application/x-cdf", "cda"},
|
||||
{"application/x-csh", "csh"},
|
||||
{"text/css", "css"},
|
||||
{"text/csv", "csv"},
|
||||
{"application/msword", "doc"},
|
||||
{"application/vnd.openxmlformats-officedocument.wordprocessingml.document", "docx"},
|
||||
{"application/vnd.ms-fontobject", "eot"},
|
||||
{"application/epub+zip", "epub"},
|
||||
{"application/gzip", "gz"},
|
||||
{"image/gif", "gif"},
|
||||
{"text/html", "htm"},
|
||||
{"image/vnd.microsoft.icon", "ico"},
|
||||
{"text/calendar", "ics"},
|
||||
{"application/java-archive", "jar"},
|
||||
{"image/jpeg", "jpg"},
|
||||
{"text/javascript", "js"},
|
||||
{"application/json", "json"},
|
||||
{"application/ld+json", "jsonld"},
|
||||
{"audio/midi audio/x-midi", "mid"},
|
||||
{"text/javascript", "mjs"},
|
||||
{"audio/mpeg", "mp3"},
|
||||
{"video/mp4", "mp4"},
|
||||
{"video/mpeg", "mpeg"},
|
||||
{"application/vnd.apple.installer+xml", "mpkg"},
|
||||
{"application/vnd.oasis.opendocument.presentation", "odp"},
|
||||
{"application/vnd.oasis.opendocument.spreadsheet", "ods"},
|
||||
{"application/vnd.oasis.opendocument.text", "odt"},
|
||||
{"audio/ogg", "oga"},
|
||||
{"video/ogg", "ogv"},
|
||||
{"application/ogg", "ogx"},
|
||||
{"audio/opus", "opus"},
|
||||
{"font/otf", "otf"},
|
||||
{"image/png", "png"},
|
||||
{"application/pdf", "pdf"},
|
||||
{"application/x-httpd-php", "php"},
|
||||
{"application/vnd.ms-powerpoint", "ppt"},
|
||||
{"application/vnd.openxmlformats-officedocument.presentationml.presentation", "pptx"},
|
||||
{"application/vnd.rar", "rar"},
|
||||
{"application/rtf", "rtf"},
|
||||
{"application/x-sh", "sh"},
|
||||
{"image/svg+xml", "svg"},
|
||||
{"application/x-tar", "tar"},
|
||||
{"image/tiff", "tif"},
|
||||
{"video/mp2t", "ts"},
|
||||
{"font/ttf", "ttf"},
|
||||
{"text/plain", "txt"},
|
||||
{"application/vnd.visio", "vsd"},
|
||||
{"audio/wav", "wav"},
|
||||
{"audio/webm", "weba"},
|
||||
{"video/webm", "webm"},
|
||||
{"image/webp", "webp"},
|
||||
{"font/woff", "woff"},
|
||||
{"font/woff2", "woff2"},
|
||||
{"application/xhtml+xml", "xhtml"},
|
||||
{"application/vnd.ms-excel", "xls"},
|
||||
{"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", "xlsx"},
|
||||
{"application/xml", "xml"},
|
||||
{"text/xml", "xml"},
|
||||
{"application/vnd.mozilla.xul+xml", "xul"},
|
||||
{"application/zip", "zip"},
|
||||
{"video/3gpp", "3gp"},
|
||||
{"audio/3gpp", "3gp"},
|
||||
{"video/3gpp2", "3g2"},
|
||||
{"audio/3gpp2", "3g2"},
|
||||
{"application/x-7z-compressed", "7z"},
|
||||
};
|
||||
|
||||
std::string ret;
|
||||
for (size_t i = 0; i < std::size(table); i++)
|
||||
{
|
||||
if (StringUtil::compareNoCase(table[i][0], content_type))
|
||||
{
|
||||
ret = table[i][1];
|
||||
break;
|
||||
}
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
Reference in New Issue
Block a user