JustAnotherArchivist
/
little-things
2
0
Bifurcation 0
Code Tickets 0 Demandes d'ajout 0 Versions 0 Wiki Activité
The little things give you away... A collection of various small helper stuff
Vous ne pouvez pas sélectionner plus de 25 sujets Les noms de sujets doivent commencer par une lettre ou un nombre, peuvent contenir des tirets ('-') et peuvent comporter jusqu'à 35 caractères.
437 Révisions
1 Branche
1.7 MiB
 
 
 
Branche: master
Branches Tags
${ item.name }
Créer la branche ${ searchTerm }
de 'master'
${ noResults }
little-things/http-response-bodies.c

390 lignes
13 KiB
Brut Lien permanent Annotations Historique 

  1. #define _GNU_SOURCE

  2. #include <ctype.h>

  3. #include <stdbool.h>

  4. #include <stdio.h>

  5. #include <stdlib.h>

  6. #include <string.h>

  7. 

  8. #ifndef BUFSIZE

  9. #define BUFSIZE 1048576

  10. #endif

  11. 

  12. #define STATE_HEADERS 0

  13. #define STATE_BODY 1  // Body with a Content-Length header

  14. #define STATE_CHUNK_LINE 2

  15. #define STATE_CHUNK_CONTENTS 3

  16. 

  17. #ifdef DEBUG

  18. #define DEBUG_PRINTF(...) do { fprintf(stderr, __VA_ARGS__); } while (false)

  19. #else

  20. #define DEBUG_PRINTF(...) do {} while (false)

  21. #endif

  22. 

  23. char* memcasemem(char* haystack, size_t haystacklen, char* needle, size_t needlelen) {

  24. 	// Case-insensitive (for ASCII) slower version of memmem

  25. 	// needle must already be in lower-case.

  26. 	if (needlelen > haystacklen) {

  27. 		// A needle longer than the haystack can never be in there.

  28. 		return NULL;

  29. 	}

  30. 	char* p1;

  31. 	char* p2;

  32. 	char* p3;

  33. 	for (p1 = haystack; p1 < haystack + haystacklen - needlelen; ++p1) {

  34. 		if (tolower((unsigned char)*p1) == *needle) {

  35. 			// Found a first char match, check the rest

  36. 			// No need to constrain p2; due to the needlelen>haystacklen check above and the limits on p1, p2 can never exceed the haystack.

  37. 			for (p2 = p1 + 1, p3 = needle + 1; p3 < needle + needlelen; p2++, p3++) {

  38. 				if (tolower((unsigned char)*p2) != *p3) {

  39. 					break;

  40. 				}

  41. 			}

  42. 			if (p3 == needle + needlelen) {

  43. 				// Full match

  44. 				return p1;

  45. 			}

  46. 		}

  47. 	}

  48. 	return NULL;

  49. }

  50. 

  51. int main(int argc, char* argv[]) {

  52. 	// Read stdin, decode HTTP responses, dump all bodies to stdout.

  53. 	// stdin may contain an extra 'URL LENGTH\n' line before each response (--meta output from warc-dump-responses).

  54. 	// One LF is inserted at the end of each response to ensure that a new response always begins on a new line.

  55. 	// If --html-fake-base is provided and the input contains URL data, every HTML response (Content-Type: text/html header) is prefixed with one line containing a fake <base> tag: <base href="URL">. The line is terminated with a LF.

  56. 	// Headers and chunk lines must fit into BUFSIZE.

  57. 	// Does not fully comply with the HTTP spec. For example, continuation lines are unsupported.

  58. 	char buf[2 * BUFSIZE];

  59. 	size_t n;

  60. 	int state = STATE_HEADERS;

  61. 	char* bufp;

  62. 	char* m0;

  63. 	char* m1;

  64. 	char* eoh;

  65. 	long int nscan;

  66. 	size_t bytes_read = 0;

  67. 	size_t length;

  68. 	bool html_fake_base = false;

  69. 	char* url = NULL; // Warning, pointer is only valid within the STATE_HEADERS block below.

  70. 	size_t urllen;

  71. 	bool have_response_length = false;

  72. 	size_t response_length;

  73. 

  74. 	if (argc == 2 && strcmp(argv[1], "--html-fake-base") == 0) {

  75. 		html_fake_base = true;

  76. 	}

  77. 

  78. 	while ((n = fread(buf, 1, BUFSIZE, stdin)) > 0) {

  79. 		bufp = buf;

  80. checkstate:

  81. 		DEBUG_PRINTF("Have %zu bytes of buffer (at %p)\n", n, (void*)bufp);

  82. 		DEBUG_PRINTF("Beginning of buffer: ");

  83. 		for (unsigned int i = 0; i < (n > 64 ? 64 : n); ++i) DEBUG_PRINTF(isprint(*(bufp + i)) ? "%c" : "\\x%02x", *(bufp + i) & 0xFF);

  84. 		DEBUG_PRINTF("\n");

  85. 		if (n == 0) {

  86. 			break;

  87. 		}

  88. 		DEBUG_PRINTF("State: %d\n", state);

  89. 		if (state == STATE_HEADERS) {

  90. 			if (n < 9) {

  91. 				fprintf(stderr, "Error: too little data before HTTP headers\n");

  92. 				return 1;

  93. 			}

  94. 			// Handle URL + Length line; optional for HTTP/1.1 but required for HTTP/1.0

  95. 			url = NULL;

  96. 			urllen = 0;

  97. 			response_length = 0;

  98. 			if (memcmp(bufp, "HTTP/1.1 ", 9) != 0 && memcmp(bufp, "HTTP/1.0 ", 9) != 0) {

  99. 				DEBUG_PRINTF("No HTTP header, looking for URL line\n");

  100. 				m0 = memmem(bufp, n, "\n", 1);

  101. 				if (!m0 || m0 == bufp) {

  102. 					fprintf(stderr, "Error: expected HTTP headers or URL line, got neither\n");

  103. 					return 1;

  104. 				}

  105. 				m1 = m0;

  106. 				// Skip back over length field

  107. 				--m0;

  108. 				while (m0 > bufp && '0' <= *m0 && *m0 <= '9') --m0;

  109. 				if (*m0 != ' ') {

  110. 					fprintf(stderr, "Error: URL line has unexpected format\n");

  111. 					return 1;

  112. 				}

  113. 				// Read length

  114. 				if (sscanf(m0, " %zu%ln", &response_length, &nscan) <= 0) {

  115. 					fprintf(stderr, "Error: URL line contains no length\n");

  116. 					return 1;

  117. 				}

  118. 				if (nscan != m1 - m0) {

  119. 					fprintf(stderr, "Error: URL line length read mismatch\n");

  120. 					return 1;

  121. 				}

  122. 				have_response_length = true;

  123. 				DEBUG_PRINTF("Response length: %zu\n", response_length);

  124. 				// Rest must now be the URL; check that there is a scheme and no CR, LF, or whitespace.

  125. 				url = bufp;

  126. 				urllen = m0 - bufp;

  127. 				if (!memmem(url, urllen, "://", 3)) {

  128. 					fprintf(stderr, "Error: URL line contains no scheme\n");

  129. 					return 1;

  130. 				}

  131. 				m0 = url;

  132. 				while (m0 < bufp + urllen && *m0 != '\r' && *m0 != '\n' && *m0 != ' ' && *m0 != '\t') ++m0;

  133. 				if (m0 != bufp + urllen) {

  134. 					fprintf(stderr, "Error: URL contains CR, LF, or whitespace\n");

  135. 					return 1;

  136. 				}

  137. 				DEBUG_PRINTF("Found URL: ");

  138. 				for (unsigned int i = 0; i < (urllen > 64 ? 64 : urllen); ++i) DEBUG_PRINTF(isprint(*(url + i)) ? "%c" : "\\x%02x", *(url + i) & 0xFF);

  139. 				if (urllen > 64) DEBUG_PRINTF("<...>");

  140. 				DEBUG_PRINTF("\n");

  141. 				// Skip over URL line and continue processing below

  142. 				DEBUG_PRINTF("Adjusting buffer pointer and n by %zu\n", m1 + 1 - bufp);

  143. 				n = n - (m1 + 1 - bufp);

  144. 				bufp = m1 + 1;

  145. 			}

  146. 			if (html_fake_base && !url) {

  147. 				fprintf(stderr, "Error: --html-fake-base requires URL lines\n");

  148. 				return 1;

  149. 			}

  150. 			if (memcmp(bufp, "HTTP/1.0 ", 9) == 0 && !have_response_length) {

  151. 				fprintf(stderr, "Error: HTTP/1.0 requires URL metadata lines\n");

  152. 				return 1;

  153. 			}

  154. 

  155. 			if (n < 9) {

  156. 				fprintf(stderr, "Error: too little data before HTTP headers\n");

  157. 				return 1;

  158. 			}

  159. 			if (memcmp(bufp, "HTTP/1.1 ", 9) == 0 || memcmp(bufp, "HTTP/1.0 ", 9) == 0) {

  160. 				// Got some headers; find transfer encoding, content length, and end of headers

  161. 				m0 = memmem(bufp, n, "\r\n\r\n", 4);

  162. 				m1 = memmem(bufp, n, "\n\n", 2);

  163. 				if (m0 && m1) {

  164. 					eoh = (m0 < m1 ? m0 + 4 : m1 + 2);

  165. 				} else if (m0) {

  166. 					eoh = m0 + 4;

  167. 				} else if (m1) {

  168. 					eoh = m1 + 2;

  169. 				} else {

  170. 					eoh = NULL;

  171. 				}

  172. 				if (!eoh) {

  173. 					fprintf(stderr, "Error: end of headers not found\n");

  174. 					return 1;

  175. 				}

  176. 				DEBUG_PRINTF("Response body begins at %p (offset %zu)\n", (void*)eoh, eoh - bufp);

  177. 

  178. 				if (memcmp(bufp, "HTTP/1.0 ", 9) == 0) {

  179. 					// HTTP 1.0 doesn't have TE, so just use the response_length for the content length and skip the other header parsing

  180. 					if (bufp + response_length < eoh) {

  181. 						fprintf(stderr, "Error: end of headers occurs after alleged response length\n");

  182. 						return 1;

  183. 					}

  184. 					length = response_length - (eoh - bufp);

  185. 					state = STATE_BODY;

  186. 				} else if ((m0 = memcasemem(bufp, eoh - bufp, "\ncontent-length:", 16)) && m0 < eoh) {

  187. 					DEBUG_PRINTF("Found Content-Length header at %p (offset %zu)\n", (void*)(m0 + 2), m0 + 2 - bufp);

  188. 					m1 = memmem(m0 + 1, eoh - (m0 + 1), "\n", 1);

  189. 					if (!m1) {

  190. 						fprintf(stderr, "Error: CRLF after Content-Length missing\n");

  191. 						return 1;

  192. 					}

  193. 					if (*(m1 - 1) == '\r') --m1;

  194. 					m0 += 16;

  195. 					while (m0 < bufp + n && (*m0 == ' ' || *m0 == '\t')) ++m0;

  196. 					if (sscanf(m0, "%zu%ln", &length, &nscan) <= 0) {

  197. 						fprintf(stderr, "Error: invalid Content-Length\n");

  198. 						return 1;

  199. 					}

  200. 					if (nscan < 0) {

  201. 						fprintf(stderr, "Error: negative nscan\n");

  202. 						return 1;

  203. 					}

  204. 					if (m0 + nscan > bufp + n) {

  205. 						fprintf(stderr, "Error: buffer overread\n");

  206. 						return 1;

  207. 					}

  208. 					m0 += nscan;

  209. 					while (m0 < bufp + n && (*m0 == ' ' || *m0 == '\t')) ++m0;

  210. 					if (m0 != m1) {

  211. 						fprintf(stderr, "Error: invalid Content-Length (noise before EOL)\n");

  212. 						return 1;

  213. 					}

  214. 					DEBUG_PRINTF("Content length: %zu\n", length);

  215. 

  216. 					state = STATE_BODY;

  217. 				} else {

  218. 					m0 = memcasemem(bufp, eoh - bufp, "\ntransfer-encoding:", 19);

  219. 					if (!m0 || m0 >= eoh) {

  220. 						DEBUG_PRINTF("No Content-Length or Transfer-Encoding, falling back to response length\n");

  221. 						if (!have_response_length) {

  222. 							fprintf(stderr, "Error: Content-Length and Transfer-Encoding missing and no response length from metadata line\n");

  223. 							return 1;

  224. 						}

  225. 						if (bufp + response_length < eoh) {

  226. 							fprintf(stderr, "Error: end of headers occurs after alleged response length\n");

  227. 							return 1;

  228. 						}

  229. 						length = response_length - (eoh - bufp);

  230. 						state = STATE_BODY;

  231. 					} else {

  232. 						DEBUG_PRINTF("Found Transfer-Encoding header at %p (offset %zu)\n", (void*)(m0 + 1), m0 + 1 - bufp);

  233. 						m1 = memmem(m0 + 1, eoh - (m0 + 1), "\n", 1);

  234. 						if (!m1 || m1 >= eoh - 1) {

  235. 							fprintf(stderr, "Error: CRLF after Transfer-Encoding missing\n");

  236. 							return 1;

  237. 						}

  238. 						m0 += 19;

  239. 						if (*(m1 - 1) == '\r') --m1;

  240. 						while (m0 < bufp + n && (*m0 == ' ' || *m0 == '\t')) ++m0;

  241. 						if (memcmp(m0, "chunked", 7) != 0) {

  242. 							fprintf(stderr, "Error: unsupported Transfer-Encoding\n");

  243. 							return 1;

  244. 						}

  245. 						m0 += 7;

  246. 						while (m0 < bufp + n && (*m0 == ' ' || *m0 == '\t')) ++m0;

  247. 						if (m0 != m1) {

  248. 							fprintf(stderr, "Error: unsupported Transfer-Encoding\n");

  249. 							return 1;

  250. 						}

  251. 						DEBUG_PRINTF("Chunked transfer encoding\n");

  252. 

  253. 						state = STATE_CHUNK_LINE;

  254. 					}

  255. 				}

  256. 

  257. 				if (html_fake_base) {

  258. 					m0 = memcasemem(bufp, eoh - bufp, "\ncontent-type:", 14);

  259. 					if (m0 && m0 < eoh) {

  260. 						DEBUG_PRINTF("Found Content-Type header at %p (offset %zu)\n", (void*)(m0 + 1), m0 + 1 - bufp);

  261. 						m1 = memmem(m0 + 1, eoh - (m0 + 1), "\n", 1);

  262. 						if (!m1) {

  263. 							fprintf(stderr, "Error: CRLF after Content-Type missing\n");

  264. 							return 1;

  265. 						}

  266. 						m0 += 14;

  267. 						while (m0 < bufp + n && (*m0 == ' ' || *m0 == '\t')) ++m0;

  268. 						if (memcmp(m0, "text/html", 9) == 0) {

  269. 							DEBUG_PRINTF("Is HTML response, inserting fake base tag\n");

  270. 							fprintf(stdout, "<base href=\"");

  271. 							fwrite(url, 1, urllen, stdout);

  272. 							fprintf(stdout, "\">\n");

  273. 						}

  274. 					}

  275. 				}

  276. 

  277. 				DEBUG_PRINTF("Adjusting buffer pointer and n by %zu\n", eoh - bufp);

  278. 				n = n - (eoh - bufp);

  279. 				bufp = eoh;

  280. 				bytes_read = 0;

  281. 				goto checkstate;

  282. 			} else {

  283. 				fprintf(stderr, "Error: expected header line, got something else\n");

  284. 				return 1;

  285. 			}

  286. 		} else if (state == STATE_BODY || state == STATE_CHUNK_CONTENTS) {

  287. 			if (length + (state == STATE_BODY ? 1 : 2) - bytes_read > n) {

  288. 				// Only got part of the body

  289. 				DEBUG_PRINTF("Partial body\n");

  290. 				DEBUG_PRINTF("Copying %zu bytes to stdout\n", n);

  291. 				fwrite(bufp, 1, n, stdout);

  292. 				bytes_read += n;

  293. 				DEBUG_PRINTF("%zu of %zu bytes from this response written\n", bytes_read, length);

  294. 			} else {

  295. 				// Remainder of the response is in the buffer. Same logic as above for small records fitting in the buffer with the headers.

  296. 				DEBUG_PRINTF("Full body\n");

  297. 				DEBUG_PRINTF("Copying %zu bytes to stdout\n", length - bytes_read);

  298. 				fwrite(bufp, 1, length - bytes_read, stdout);

  299. 				if (state == STATE_BODY) {

  300. 					fprintf(stdout, "\n");

  301. 				}

  302. 				if (state == STATE_CHUNK_CONTENTS && *(bufp + length - bytes_read) == '\r') {

  303. 					// Stupid hack to enforce the CRLF

  304. 					++length;

  305. 				}

  306. 				if (memcmp(bufp + length - bytes_read, "\n", 1) != 0) {

  307. 					fprintf(stderr, "Error: end of HTTP body not found\n");

  308. 					return 1;

  309. 				}

  310. 				DEBUG_PRINTF("Adjusting buffer pointer and n by %zu\n", length + 1 - bytes_read);

  311. 				n = n - (length + 1 - bytes_read);

  312. 				bufp = bufp + length + 1 - bytes_read;

  313. 				if (n < BUFSIZE) {

  314. 					DEBUG_PRINTF("Buffer too small (%zu bytes), moving and refilling\n", n);

  315. 					memmove(buf, bufp, n);

  316. 					bufp = buf;

  317. 					n += fread(buf + n, 1, BUFSIZE, stdin);

  318. 				}

  319. 				if (state == STATE_BODY) {

  320. 					state = STATE_HEADERS;

  321. 				} else {

  322. 					state = STATE_CHUNK_LINE;

  323. 				}

  324. 				goto checkstate;

  325. 			}

  326. 		} else if (state == STATE_CHUNK_LINE) {

  327. 			m1 = memmem(bufp, n, "\r\n", 2);

  328. 			if (!m1) {

  329. 				fprintf(stderr, "Error: chunk line EOL missing\n");

  330. 				return 1;

  331. 			}

  332. 			m0 = bufp;

  333. 			while (m0 < bufp + n && (*m0 == ' ' || *m0 == '\t')) ++m0;

  334. 			if (sscanf(m0, "%zx%ln", &length, &nscan) <= 0) {

  335. 				fprintf(stderr, "Error: invalid chunk length\n");

  336. 				return 1;

  337. 			}

  338. 			if (nscan < 0) {

  339. 				fprintf(stderr, "Error: negative nscan\n");

  340. 				return 1;

  341. 			}

  342. 			if (m0 + nscan > bufp + n) {

  343. 				fprintf(stderr, "Error: buffer overread\n");

  344. 				return 1;

  345. 			}

  346. 			m0 += nscan;

  347. 			while (m0 < m1 && (*m0 == ' ' || *m0 == '\t')) ++m0;

  348. 			if (*m0 != ';' && m0 != m1) {

  349. 				fprintf(stderr, "Error: invalid Content-Length (noise before EOL)\n");

  350. 				return 1;

  351. 			}

  352. 			DEBUG_PRINTF("Chunk length: %zu bytes\n", length);

  353. 

  354. 			DEBUG_PRINTF("Adjusting buffer pointer and n by %zu\n", m1 + 2 - bufp);

  355. 			n = n - (m1 + 2 - bufp);

  356. 			bufp = m1 + 2;

  357. 

  358. 			if (length == 0) {

  359. 				// End of response, must be followed by CRLF + LF

  360. 				if (n < 3) {

  361. 					fprintf(stderr, "Error: buffer exhausted while looking for empty chunk CRLF\n");

  362. 					return 1;

  363. 				}

  364. 				if (*(m1 + 2) != '\r' || *(m1 + 3) != '\n' || *(m1 + 4) != '\n') {

  365. 					fprintf(stderr, "Error: end of HTTP body not found\n");

  366. 					return 1;

  367. 				}

  368. 				fprintf(stdout, "\n");

  369. 				n -= 3;

  370. 				bufp += 3;

  371. 				state = STATE_HEADERS;

  372. 			} else {

  373. 				state = STATE_CHUNK_CONTENTS;

  374. 			}

  375. 

  376. 			if (n < BUFSIZE) {

  377. 				DEBUG_PRINTF("Buffer too small (%zu bytes), moving and refilling\n", n);

  378. 				memmove(buf, bufp, n);

  379. 				bufp = buf;

  380. 				n += fread(buf + n, 1, BUFSIZE, stdin);

  381. 			}

  382. 			goto checkstate;

  383. 		}

  384. 	}

  385. 	if (state != STATE_HEADERS) {

  386. 		fprintf(stderr, "Error: incomplete body at the end of input\n");

  387. 		return 1;

  388. 	}

  389. }