1#include "fastfetch.h"
2#include "common/networking.h"
3#include "common/time.h"
4#include "common/library.h"
5#include "common/strutil.h"
6#include "common/mallocHelper.h"
7#include "common/debug.h"
8
9#include <unistd.h>
10#include <poll.h>
11#include <sys/time.h>
12#include <sys/socket.h>
13#include <netdb.h>
14#include <netinet/in.h> // For FreeBSD
15#include <netinet/tcp.h>
16#include <errno.h>
17#include <fcntl.h>
18
19// Upper bound of a single HTTP response, guarding against excessive memory allocation
20#define FF_NETWORKING_MAX_RESPONSE_SIZE (1024u * 1024u)
21
22static const char* tryNonThreadingFastPath(FFNetworkingState* state) {
23#if defined(TCP_FASTOPEN) || __APPLE__
24
25 if (!state->tfo) {
26 #if __linux__ || __GNU__
27 // Linux doesn't support sendto() on unconnected sockets
28 FF_DEBUG("TCP Fast Open disabled, skipping");
29 return "TCP Fast Open disabled";
30 #endif
31 } else {
32 FF_DEBUG("Attempting to use TCP Fast Open to connect");
33
34 #ifndef __APPLE__ // On macOS, TCP_FASTOPEN doesn't seem to be needed
35 // Set TCP Fast Open
36 int flag = 1;
37 if (setsockopt(state->sockfd, IPPROTO_TCP,
38 #ifdef __APPLE__
39 // https://github.com/rust-lang/libc/pull/3135
40 0x218 // TCP_FASTOPEN_FORCE_ENABLE
41 #else
42 TCP_FASTOPEN
43 #endif
44 ,
45 &flag,
46 sizeof(flag)) != 0) {
47 FF_DEBUG("Failed to set TCP_FASTOPEN option: %s", strerror(errno));
48 return "setsockopt(TCP_FASTOPEN) failed";
49 } else {
50 #if __linux__ || __GNU__
51 FF_DEBUG("Successfully set TCP_FASTOPEN option, queue length: %d", flag);
52 #elif defined(__APPLE__)
53 FF_DEBUG("Successfully set TCP_FASTOPEN_FORCE_ENABLE option");
54 #else
55 FF_DEBUG("Successfully set TCP_FASTOPEN option");
56 #endif
57 }
58 #endif
59 }
60
61 #ifndef __APPLE__
62 FF_DEBUG("Using sendto() "
63 #ifdef MSG_FASTOPEN
64 "+ MSG_FASTOPEN "
65 #endif
66 #ifdef MSG_NOSIGNAL
67 "+ MSG_NOSIGNAL "
68 #endif
69 "to send %u bytes of data", state->command.length);
70 ssize_t sent = sendto(state->sockfd,
71 state->command.chars,
72 state->command.length,
73 #ifdef MSG_FASTOPEN
74 MSG_FASTOPEN |
75 #endif
76 #ifdef MSG_NOSIGNAL
77 MSG_NOSIGNAL |
78 #endif
79 0,
80 state->addr->ai_addr,
81 state->addr->ai_addrlen);
82 #else
83 if (fcntl(state->sockfd, F_SETFL, O_NONBLOCK) == -1) {
84 FF_DEBUG("fcntl(F_SETFL) failed: %s", strerror(errno));
85 return "fcntl(F_SETFL) failed";
86 }
87 FF_DEBUG("Using connectx() to send %u bytes of data", state->command.length);
88 // Use connectx to establish connection and send data in one call
89 size_t sent;
90 if (connectx(state->sockfd,
91 &(sa_endpoints_t) {
92 .sae_dstaddr = state->addr->ai_addr,
93 .sae_dstaddrlen = state->addr->ai_addrlen,
94 },
95 SAE_ASSOCID_ANY,
96 state->tfo ? CONNECT_DATA_IDEMPOTENT : 0,
97 &(struct iovec) {
98 .iov_base = state->command.chars,
99 .iov_len = state->command.length,
100 },
101 1,
102 &sent,
103 nullptr) != 0) {
104 sent = 0;
105 }
106 if (fcntl(state->sockfd, F_SETFL, 0) == -1) {
107 FF_DEBUG("fcntl(F_SETFL) failed: %s", strerror(errno));
108 return "fcntl(F_SETFL) failed";
109 }
110 #endif
111 if (sent > 0 || (errno == EAGAIN || errno == EWOULDBLOCK
112 #ifdef __APPLE__
113 // On macOS EINPROGRESS means the connection cannot be completed immediately
114 // On Linux, it means the TFO cookie is not available locally
115 || errno == EINPROGRESS
116 #endif
117 )) {
118 FF_DEBUG(
119 #ifdef __APPLE__
120 "connectx()"
121 #else
122 "sendto()"
123 #endif
124 " %s (sent=%zd, %s)",
125 errno == 0 ? "succeeded" : "was in progress",
126 sent,
127 strerror(errno));
128 freeaddrinfo(state->addr);
129 state->addr = nullptr;
130 ffStrbufDestroy(&state->command);
131 return nullptr;
132 }
133
134 FF_DEBUG(
135 #ifdef __APPLE__
136 "connectx()"
137 #else
138 "sendto()"
139 #endif
140 " failed: %s",
141 strerror(errno));
142 #ifdef __APPLE__
143 return "connectx() failed";
144 #else
145 return "sendto() failed";
146 #endif
147#else
148 FF_UNUSED(state);
149 FF_DEBUG("TFO support is not available");
150 return "TFO support is not available";
151#endif
152}
153
154// Traditional connect and send function
155static const char* connectAndSend(FFNetworkingState* state) {
156 const char* ret = nullptr;
157 FF_DEBUG("Using traditional connection method to connect");
158
159 FF_DEBUG("Attempting connect() to server...");
160 if (connect(state->sockfd, state->addr->ai_addr, state->addr->ai_addrlen) == -1) {
161 FF_DEBUG("connect() failed: %s", strerror(errno));
162 ret = "connect() failed";
163 goto error;
164 }
165 FF_DEBUG("connect() succeeded");
166
167 FF_DEBUG("Attempting to send %u bytes of data...", state->command.length);
168 if (send(state->sockfd, state->command.chars, state->command.length, 0) < 0) {
169 FF_DEBUG("send() failed: %s", strerror(errno));
170 ret = "send() failed";
171 goto error;
172 }
173 FF_DEBUG("Data sent successfully");
174
175 goto exit;
176
177error:
178 FF_DEBUG("Error occurred, closing socket");
179 close(state->sockfd);
180 state->sockfd = -1;
181
182exit:
183 FF_DEBUG("Releasing address info and other resources");
184 freeaddrinfo(state->addr);
185 state->addr = nullptr;
186 ffStrbufDestroy(&state->command);
187
188 return ret;
189}
190
191FF_THREAD_ENTRY_DECL_WRAPPER(connectAndSend, FFNetworkingState*);
192
193// Parallel DNS resolution and socket creation
194static const char* initNetworkingState(FFNetworkingState* state, const char* host, uint16_t port, const char* path, const char* headers) {
195 FF_DEBUG("Initializing network connection state: host=%s, path=%s", host, path);
196
197 // Initialize command and host information
198 ffStrbufInitA(&state->command, 128);
199 ffStrbufAppendS(&state->command, "GET ");
200 ffStrbufAppendS(&state->command, path);
201 ffStrbufAppendS(&state->command, " HTTP/1.0\r\nHost: ");
202 if (strchr(host, ':') != nullptr) {
203 // An IPv6 literal has to be bracketed in the Host header (RFC 9110 7.2), while
204 // getaddrinfo() wants it bare
205 ffStrbufAppendC(&state->command, '[');
206 ffStrbufAppendS(&state->command, host);
207 ffStrbufAppendC(&state->command, ']');
208 } else {
209 ffStrbufAppendS(&state->command, host);
210 }
211 // The Host header carries the port whenever it is not the default one (RFC 9110 7.2)
212 if (port != 80) {
213 ffStrbufAppendF(&state->command, ":%u", port);
214 }
215 ffStrbufAppendS(&state->command, "\r\nConnection: close\r\n"); // Explicitly tell the server we don't need to keep the connection
216
217 ffStrbufAppendS(&state->command, headers);
218 ffStrbufAppendS(&state->command, "\r\n");
219
220#ifdef FF_HAVE_THREADS
221 state->thread = 0;
222 FF_DEBUG("Thread ID initialized to 0");
223#endif
224
225 const char* ret = nullptr;
226
227 struct addrinfo hints = {
228 .ai_family = state->ipv6 ? AF_INET6 : AF_INET,
229 .ai_socktype = SOCK_STREAM,
230 .ai_flags = AI_NUMERICSERV
231 };
232
233 char portA[6];
234 snprintf(portA, sizeof(portA), "%u", port);
235
236 FF_DEBUG("Resolving address: %s:%u (%s)", host, port, state->ipv6 ? "IPv6" : "IPv4");
237 // Use AI_NUMERICSERV flag to indicate the service is a numeric port, reducing parsing time
238
239 int gaiRes = getaddrinfo(host, portA, &hints, &state->addr);
240 if (gaiRes != 0) {
241 FF_DEBUG("getaddrinfo() failed: %s (res=%d)", gai_strerror(gaiRes), gaiRes);
242 ret = "getaddrinfo() failed";
243 goto error;
244 }
245 FF_DEBUG("Address resolution successful");
246
247 FF_DEBUG("Creating socket");
248 state->sockfd = socket(state->addr->ai_family, state->addr->ai_socktype, state->addr->ai_protocol);
249 if (state->sockfd == -1) {
250 FF_DEBUG("socket() failed: %s", strerror(errno));
251 ret = "socket() failed";
252 goto error;
253 }
254 FF_DEBUG("Socket creation successful: fd=%d", state->sockfd);
255
256 int flag = 1;
257#ifdef TCP_NODELAY
258 // Disable Nagle's algorithm to reduce small packet transmission delay
259 if (setsockopt(state->sockfd, IPPROTO_TCP, TCP_NODELAY, &flag, sizeof(flag)) != 0) {
260 FF_DEBUG("Failed to set TCP_NODELAY: %s", strerror(errno));
261 } else {
262 FF_DEBUG("Successfully disabled Nagle's algorithm");
263 }
264#endif
265
266#ifdef TCP_QUICKACK
267 // Set TCP_QUICKACK option to avoid delayed acknowledgments
268 if (setsockopt(state->sockfd, IPPROTO_TCP, TCP_QUICKACK, &flag, sizeof(flag)) != 0) {
269 FF_DEBUG("Failed to set TCP_QUICKACK: %s", strerror(errno));
270 } else {
271 FF_DEBUG("Successfully enabled TCP quick acknowledgment");
272 }
273#endif
274
275#ifdef SO_NOSIGPIPE
276 // Prevent SIGPIPE when the server closes the connection during write
277 if (setsockopt(state->sockfd, SOL_SOCKET, SO_NOSIGPIPE, &flag, sizeof(flag)) != 0) {
278 FF_DEBUG("Failed to set SO_NOSIGPIPE: %s", strerror(errno));
279 } else {
280 FF_DEBUG("Successfully set SO_NOSIGPIPE");
281 }
282#endif
283
284 if (state->timeout > 0) {
285 FF_DEBUG("Setting connection timeout: %u ms", state->timeout);
286 [[maybe_unused]] uint32_t sec = state->timeout / 1000;
287 if (sec == 0) {
288 sec = 1;
289 }
290
291#ifdef TCP_CONNECTIONTIMEOUT
292 FF_DEBUG("Using TCP_CONNECTIONTIMEOUT: %u seconds", sec);
293 setsockopt(state->sockfd, IPPROTO_TCP, TCP_CONNECTIONTIMEOUT, &sec, sizeof(sec));
294#elif defined(TCP_KEEPINIT)
295 FF_DEBUG("Using TCP_KEEPINIT: %u seconds", sec);
296 setsockopt(state->sockfd, IPPROTO_TCP, TCP_KEEPINIT, &sec, sizeof(sec));
297#elif defined(TCP_USER_TIMEOUT)
298 FF_DEBUG("Using TCP_USER_TIMEOUT: %u milliseconds", state->timeout);
299 setsockopt(state->sockfd, IPPROTO_TCP, TCP_USER_TIMEOUT, &state->timeout, sizeof(state->timeout));
300#else
301 FF_DEBUG("Current platform does not support TCP connection timeout");
302#endif
303 }
304
305 return nullptr;
306
307error:
308 FF_DEBUG("Error occurred during initialization");
309 if (state->addr != nullptr) {
310 FF_DEBUG("Releasing address information");
311 freeaddrinfo(state->addr);
312 state->addr = nullptr;
313 }
314
315 if (state->sockfd > 0) {
316 FF_DEBUG("Closing socket: fd=%d", state->sockfd);
317 close(state->sockfd);
318 state->sockfd = -1;
319 }
320
321 ffStrbufClear(&state->command);
322 return ret;
323}
324
325const char* ffNetworkingSendHttpRequest(FFNetworkingState* state, const char* host, uint16_t port, const char* path, const char* headers) {
326 FF_DEBUG("Preparing to send HTTP request: host=%s, port=%u, path=%s", host, port, path);
327
328 const char* initResult = initNetworkingState(state, host, port, path, headers);
329 if (initResult != nullptr) {
330 FF_DEBUG("Initialization failed: %s", initResult);
331 return initResult;
332 }
333 FF_DEBUG("Network state initialization successful");
334
335 const char* tfoResult = tryNonThreadingFastPath(state);
336 if (tfoResult == nullptr) {
337 FF_DEBUG("TryNonThreadingFastPath() succeeded or in progress");
338 return nullptr;
339 }
340 FF_DEBUG("TryNonThreadingFastPath() failed: %s, trying traditional connection", tfoResult);
341
342#ifdef FF_HAVE_THREADS
343 if (instance.config.general.multithreading) {
344 FF_DEBUG("Multithreading mode enabled, creating connection thread");
345 state->thread = ffThreadCreate(connectAndSendThreadMain, state);
346 if (state->thread) {
347 FF_DEBUG("Thread creation successful: thread=%p", (void*) (uintptr_t) state->thread);
348 return nullptr;
349 }
350 FF_DEBUG("Thread creation failed");
351 } else {
352 FF_DEBUG("Multithreading mode disabled, connecting in main thread");
353 }
354#endif
355
356 return connectAndSend(state);
357}
358
359const char* ffNetworkingRecvHttpResponse(FFNetworkingState* state, FFstrbuf* buffer) {
360 assert(buffer->allocated > 0);
361 FF_DEBUG("Preparing to receive HTTP response");
362 uint32_t timeout = state->timeout;
363
364#ifdef FF_HAVE_THREADS
365 if (state->thread) {
366 FF_DEBUG("Connection thread is running, waiting for it to complete (timeout=%u ms)", timeout);
367 if (!ffThreadJoin(state->thread, timeout)) {
368 FF_DEBUG("Thread join failed or timed out");
369 return "ffThreadJoin() failed or timeout";
370 }
371 FF_DEBUG("Thread completed successfully");
372 state->thread = 0;
373 }
374#endif
375
376 if (state->sockfd == -1) {
377 FF_DEBUG("Invalid socket, HTTP request might have failed");
378 return "ffNetworkingSendHttpRequest() failed";
379 }
380
381 // Set larger initial receive buffer instead of small repeated receives
382 int rcvbuf = 64 * 1024;
383 setsockopt(state->sockfd, SOL_SOCKET, SO_RCVBUF, &rcvbuf, sizeof(rcvbuf));
384
385 // The timeout has to be enforced by the socket itself on every platform: the poll() below
386 // only reports readability once, so a server that sends a partial response and then keeps
387 // the connection open would otherwise block this loop forever.
388 if (timeout > 0) {
389 FF_DEBUG("Setting receive timeout: %u ms", timeout);
390 struct timeval timev;
391 timev.tv_sec = timeout / 1000;
392 timev.tv_usec = (typeof(timev.tv_usec)) ((timeout % 1000) * 1000); // milliseconds to microseconds
393 setsockopt(state->sockfd, SOL_SOCKET, SO_RCVTIMEO, &timev, sizeof(timev));
394 }
395
396#ifdef __APPLE__
397 // poll for the socket to be readable.
398 // Because of the non-blocking connectx() call, the connection might not be established yet
399 FF_DEBUG("Using poll() to check if socket is readable");
400 {
401 int pollRes = poll(&(struct pollfd) {
402 .fd = state->sockfd,
403 .events = POLLIN },
404 1,
405 timeout > 0 ? (int) timeout : -1);
406 if (pollRes == 0) {
407 FF_DEBUG("poll() timed out after %u ms", timeout);
408 close(state->sockfd);
409 state->sockfd = -1;
410 return "poll() timeout";
411 } else if (pollRes == -1) {
412 FF_DEBUG("poll() failed: %s", strerror(errno));
413 close(state->sockfd);
414 state->sockfd = -1;
415 return "poll() failed";
416 }
417 }
418 FF_DEBUG("Socket is readable, proceeding to receive data");
419#endif
420
421 if (shutdown(state->sockfd, SHUT_WR) == -1) {
422 FF_DEBUG("Failed to shutdown socket send: %s", strerror(errno));
423 // Not a critical error, continue anyway
424 }
425
426 FF_DEBUG("Starting data reception");
427 [[maybe_unused]] int recvCount = 0;
428 uint32_t contentLength = 0;
429 uint32_t headerEnd = 0;
430 bool chunked = false;
431
432 // Runs until the response is framed; the buffer is grown on demand at the top
433 for (;;) {
434 if (ffStrbufGetFree(buffer) == 0) {
435 // `Content-Length` may be absent (e.g. chunked responses). Grow the buffer
436 // on demand instead of silently truncating the response.
437 if (buffer->allocated >= FF_NETWORKING_MAX_RESPONSE_SIZE) {
438 FF_DEBUG("Response is too large: %u bytes, aborting", buffer->allocated);
439 close(state->sockfd);
440 state->sockfd = -1;
441 return "Response too large";
442 }
443 FF_DEBUG("Receive buffer is full, extending it");
444 // Asking for exactly the room that is left under the cap, rather than for as much as
445 // the buffer holds again: the allocation is rounded up to a power of two, so a doubling
446 // from a size that is not one of those lands past the cap the check above just cleared.
447 ffStrbufEnsureFreeNoCheck(buffer, FF_NETWORKING_MAX_RESPONSE_SIZE - buffer->length - 1);
448 }
449
450 // When the remaining length is known, ask for exactly that much. MSG_WAITALL then
451 // returns as soon as the response is complete, instead of waiting for the server
452 // to close the connection.
453 // Without a Content-Length the requested length is just "whatever fits", so
454 // waiting for all of it would block until the server closes -- and the framing
455 // checks below would never get a chance to run. Read whatever has arrived instead
456 // and let those checks decide when the response is complete.
457 uint32_t want = ffStrbufGetFree(buffer);
458 int recvFlags = 0;
459 if (contentLength > 0 && headerEnd > 0) {
460 uint32_t remaining = headerEnd + 4 + contentLength - buffer->length;
461 if (remaining < want) {
462 want = remaining;
463 }
464 recvFlags = MSG_WAITALL;
465 }
466
467 FF_DEBUG("Data reception loop #%d, current buffer size: %u, requesting %u bytes",
468 ++recvCount,
469 buffer->length,
470 want);
471
472 ssize_t received = recv(state->sockfd, buffer->chars + buffer->length, want, recvFlags);
473
474 if (received <= 0) {
475 if (received == 0) {
476 FF_DEBUG("Connection closed (received=0)");
477 } else {
478 FF_DEBUG("Reception failed: %s", strerror(errno));
479 }
480 break;
481 }
482
483 buffer->length += (uint32_t) received;
484 buffer->chars[buffer->length] = '\0';
485
486 FF_DEBUG("Successfully received %zd bytes of data, total: %u bytes", received, buffer->length);
487
488 // Check if HTTP header end marker is found
489 if (headerEnd == 0) {
490 char* pHeaderEnd = memmem(buffer->chars, buffer->length, "\r\n\r\n", 4);
491 if (pHeaderEnd) {
492 headerEnd = (uint32_t) (pHeaderEnd - buffer->chars);
493 FF_DEBUG("Found HTTP header end marker, position: %u", headerEnd);
494
495 // Check for Content-Length header to pre-allocate enough memory
496 uint32_t valueLen = 0;
497 const char* clHeader = ffNetworkingFindHeader(buffer->chars, headerEnd, "Content-Length:", &valueLen);
498 if (clHeader) {
499 contentLength = (uint32_t) strtoul(clHeader, nullptr, 10);
500 if (contentLength > 0) {
501 if (contentLength > FF_NETWORKING_MAX_RESPONSE_SIZE) { // 1MB limit to prevent excessive memory allocation and potential attacks
502 FF_DEBUG("Content-Length is too large: %u bytes, aborting", contentLength);
503 close(state->sockfd);
504 state->sockfd = -1;
505 return "Content-Length too large";
506 }
507
508 FF_DEBUG("Detected Content-Length: %u, pre-allocating buffer", contentLength);
509 // Ensure buffer is large enough, adding header size and some margin
510 ffStrbufEnsureFree(buffer, contentLength + 16);
511 FF_DEBUG("Extended receive buffer to %u bytes", buffer->allocated);
512 }
513 }
514
515 // A chunked response has no Content-Length; it is framed by a last-chunk
516 const char* teHeader = ffNetworkingFindHeader(buffer->chars, headerEnd, "Transfer-Encoding:", &valueLen);
517 if (teHeader != nullptr) {
518 switch (ffNetworkingParseTransferEncoding(teHeader, valueLen)) {
519 case FF_NETWORKING_TE_CHUNKED:
520 FF_DEBUG("Detected chunked transfer encoding");
521 chunked = true;
522 break;
523 case FF_NETWORKING_TE_UNSUPPORTED:
524 // The framing of e.g. `gzip, chunked` is unreadable and the payload
525 // would stay encoded, so fail instead of returning garbage
526 FF_DEBUG("Unsupported Transfer-Encoding: %.*s", (int) valueLen, teHeader);
527 close(state->sockfd);
528 state->sockfd = -1;
529 return "Unsupported Transfer-Encoding";
530 default:
531 break;
532 }
533 }
534 }
535 }
536
537 // Stop as soon as the response is framed, rather than waiting for the FIN
538 if (chunked) {
539 uint32_t consumed = 0;
540 int complete = ffNetworkingChunkedComplete(buffer->chars + headerEnd + 4, buffer->length - headerEnd - 4, &consumed);
541 if (complete < 0) {
542 FF_DEBUG("Malformed chunked body");
543 close(state->sockfd);
544 state->sockfd = -1;
545 return "Malformed chunked body";
546 }
547 if (complete > 0) {
548 FF_DEBUG("Chunked body complete, %u bytes of encoded body", consumed);
549 break;
550 }
551 } else if (contentLength > 0 && buffer->length >= headerEnd + 4 + contentLength) {
552 break;
553 }
554 }
555
556 FF_DEBUG("Closing socket: fd=%d", state->sockfd);
557 close(state->sockfd);
558 state->sockfd = -1;
559
560 if (buffer->length == 0) {
561 FF_DEBUG("Server response is empty");
562 return "Empty server response received";
563 }
564
565 if (headerEnd == 0) {
566 FF_DEBUG("No HTTP header end marker found");
567 return "No HTTP header end found";
568 }
569
570 if (chunked && !ffNetworkingDecodeChunked(buffer, &headerEnd)) {
571 FF_DEBUG("Failed to decode chunked response");
572 return "Failed to decode chunked response";
573 }
574
575 if (!ffStrbufStartsWithS(buffer, "HTTP/1.0 200 OK\r\n") && !ffStrbufStartsWithS(buffer, "HTTP/1.1 200 OK\r\n")) {
576 FF_DEBUG("Invalid response: %.40s...", buffer->chars);
577 return "Invalid response";
578 }
579 FF_DEBUG("Received valid HTTP 200 response, content %u bytes, total %u bytes", contentLength, buffer->length);
580
581 // A chunked response was framed by its last-chunk and has been rewritten around it, so any
582 // `Content-Length` the server also sent describes a body that no longer exists. RFC 9112 6.3
583 // says not to send both; when one does, the chunked framing is the one that was acted on.
584 if (!chunked && contentLength > 0 && buffer->length != contentLength + headerEnd + 4) {
585 FF_DEBUG("Received content length mismatches: %u != %u", buffer->length, contentLength + headerEnd + 4);
586 return "Content length mismatch";
587 }
588
589 return nullptr;
590}