JustAnotherArchivist
/
little-things
2
0
Fork 0
Code Issues 0 Pull Requests 0 Releases 0 Wiki Activity
The little things give you away... A collection of various small helper stuff
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
437 Commits
1 Branch
1.7 MiB
 
 
 
Branch: master
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'master'
${ noResults }
little-things/http-response-bodies.c

390 lines
13 KiB
Raw Permalink Blame History 

  1. #define _GNU_SOURCE

  2. #include <ctype.h>

  3. #include <stdbool.h>

  4. #include <stdio.h>

  5. #include <stdlib.h>

  6. #include <string.h>

  7. 

  8. #ifndef BUFSIZE

  9. #define BUFSIZE 1048576

  10. #endif

  11. 

  12. #define STATE_HEADERS 0

  13. #define STATE_BODY 1  // Body with a Content-Length header

  14. #define STATE_CHUNK_LINE 2

  15. #define STATE_CHUNK_CONTENTS 3

  16. 

  17. #ifdef DEBUG

  18. #define DEBUG_PRINTF(...) do { fprintf(stderr, __VA_ARGS__); } while (false)

  19. #else

  20. #define DEBUG_PRINTF(...) do {} while (false)

  21. #endif

  22. 

  23. char* memcasemem(char* haystack, size_t haystacklen, char* needle, size_t needlelen) {

  24. 	// Case-insensitive (for ASCII) slower version of memmem

  25. 	// needle must already be in lower-case.

  26. 	if (needlelen > haystacklen) {

  27. 		// A needle longer than the haystack can never be in there.

  28. 		return NULL;

  29. 	}

  30. 	char* p1;

  31. 	char* p2;

  32. 	char* p3;

  33. 	for (p1 = haystack; p1 < haystack + haystacklen - needlelen; ++p1) {

  34. 		if (tolower((unsigned char)*p1) == *needle) {

  35. 			// Found a first char match, check the rest

  36. 			// No need to constrain p2; due to the needlelen>haystacklen check above and the limits on p1, p2 can never exceed the haystack.

  37. 			for (p2 = p1 + 1, p3 = needle + 1; p3 < needle + needlelen; p2++, p3++) {

  38. 				if (tolower((unsigned char)*p2) != *p3) {

  39. 					break;

  40. 				}

  41. 			}

  42. 			if (p3 == needle + needlelen) {

  43. 				// Full match

  44. 				return p1;

  45. 			}

  46. 		}

  47. 	}

  48. 	return NULL;

  49. }

  50. 

  51. int main(int argc, char* argv[]) {

  52. 	// Read stdin, decode HTTP responses, dump all bodies to stdout.

  53. 	// stdin may contain an extra 'URL LENGTH\n' line before each response (--meta output from warc-dump-responses).

  54. 	// One LF is inserted at the end of each response to ensure that a new response always begins on a new line.

  55. 	// If --html-fake-base is provided and the input contains URL data, every HTML response (Content-Type: text/html header) is prefixed with one line containing a fake <base> tag: <base href="URL">. The line is terminated with a LF.

  56. 	// Headers and chunk lines must fit into BUFSIZE.

  57. 	// Does not fully comply with the HTTP spec. For example, continuation lines are unsupported.

  58. 	char buf[2 * BUFSIZE];

  59. 	size_t n;

  60. 	int state = STATE_HEADERS;

  61. 	char* bufp;

  62. 	char* m0;

  63. 	char* m1;

  64. 	char* eoh;

  65. 	long int nscan;

  66. 	size_t bytes_read = 0;

  67. 	size_t length;

  68. 	bool html_fake_base = false;

  69. 	char* url = NULL; // Warning, pointer is only valid within the STATE_HEADERS block below.

  70. 	size_t urllen;

  71. 	bool have_response_length = false;

  72. 	size_t response_length;

  73. 

  74. 	if (argc == 2 && strcmp(argv[1], "--html-fake-base") == 0) {

  75. 		html_fake_base = true;

  76. 	}

  77. 

  78. 	while ((n = fread(buf, 1, BUFSIZE, stdin)) > 0) {

  79. 		bufp = buf;

  80. checkstate:

  81. 		DEBUG_PRINTF("Have %zu bytes of buffer (at %p)\n", n, (void*)bufp);

  82. 		DEBUG_PRINTF("Beginning of buffer: ");

  83. 		for (unsigned int i = 0; i < (n > 64 ? 64 : n); ++i) DEBUG_PRINTF(isprint(*(bufp + i)) ? "%c" : "\\x%02x", *(bufp + i) & 0xFF);

  84. 		DEBUG_PRINTF("\n");

  85. 		if (n == 0) {

  86. 			break;

  87. 		}

  88. 		DEBUG_PRINTF("State: %d\n", state);

  89. 		if (state == STATE_HEADERS) {

  90. 			if (n < 9) {

  91. 				fprintf(stderr, "Error: too little data before HTTP headers\n");

  92. 				return 1;

  93. 			}

  94. 			// Handle URL + Length line; optional for HTTP/1.1 but required for HTTP/1.0

  95. 			url = NULL;

  96. 			urllen = 0;

  97. 			response_length = 0;

  98. 			if (memcmp(bufp, "HTTP/1.1 ", 9) != 0 && memcmp(bufp, "HTTP/1.0 ", 9) != 0) {

  99. 				DEBUG_PRINTF("No HTTP header, looking for URL line\n");

  100. 				m0 = memmem(bufp, n, "\n", 1);

  101. 				if (!m0 || m0 == bufp) {

  102. 					fprintf(stderr, "Error: expected HTTP headers or URL line, got neither\n");

  103. 					return 1;

  104. 				}

  105. 				m1 = m0;

  106. 				// Skip back over length field

  107. 				--m0;

  108. 				while (m0 > bufp && '0' <= *m0 && *m0 <= '9') --m0;

  109. 				if (*m0 != ' ') {

  110. 					fprintf(stderr, "Error: URL line has unexpected format\n");

  111. 					return 1;

  112. 				}

  113. 				// Read length

  114. 				if (sscanf(m0, " %zu%ln", &response_length, &nscan) <= 0) {

  115. 					fprintf(stderr, "Error: URL line contains no length\n");

  116. 					return 1;

  117. 				}

  118. 				if (nscan != m1 - m0) {

  119. 					fprintf(stderr, "Error: URL line length read mismatch\n");

  120. 					return 1;

  121. 				}

  122. 				have_response_length = true;

  123. 				DEBUG_PRINTF("Response length: %zu\n", response_length);

  124. 				// Rest must now be the URL; check that there is a scheme and no CR, LF, or whitespace.

  125. 				url = bufp;

  126. 				urllen = m0 - bufp;

  127. 				if (!memmem(url, urllen, "://", 3)) {

  128. 					fprintf(stderr, "Error: URL line contains no scheme\n");

  129. 					return 1;

  130. 				}

  131. 				m0 = url;

  132. 				while (m0 < bufp + urllen && *m0 != '\r' && *m0 != '\n' && *m0 != ' ' && *m0 != '\t') ++m0;

  133. 				if (m0 != bufp + urllen) {

  134. 					fprintf(stderr, "Error: URL contains CR, LF, or whitespace\n");

  135. 					return 1;

  136. 				}

  137. 				DEBUG_PRINTF("Found URL: ");

  138. 				for (unsigned int i = 0; i < (urllen > 64 ? 64 : urllen); ++i) DEBUG_PRINTF(isprint(*(url + i)) ? "%c" : "\\x%02x", *(url + i) & 0xFF);

  139. 				if (urllen > 64) DEBUG_PRINTF("<...>");

  140. 				DEBUG_PRINTF("\n");

  141. 				// Skip over URL line and continue processing below

  142. 				DEBUG_PRINTF("Adjusting buffer pointer and n by %zu\n", m1 + 1 - bufp);

  143. 				n = n - (m1 + 1 - bufp);

  144. 				bufp = m1 + 1;

  145. 			}

  146. 			if (html_fake_base && !url) {

  147. 				fprintf(stderr, "Error: --html-fake-base requires URL lines\n");

  148. 				return 1;

  149. 			}

  150. 			if (memcmp(bufp, "HTTP/1.0 ", 9) == 0 && !have_response_length) {

  151. 				fprintf(stderr, "Error: HTTP/1.0 requires URL metadata lines\n");

  152. 				return 1;

  153. 			}

  154. 

  155. 			if (n < 9) {

  156. 				fprintf(stderr, "Error: too little data before HTTP headers\n");

  157. 				return 1;

  158. 			}

  159. 			if (memcmp(bufp, "HTTP/1.1 ", 9) == 0 || memcmp(bufp, "HTTP/1.0 ", 9) == 0) {

  160. 				// Got some headers; find transfer encoding, content length, and end of headers

  161. 				m0 = memmem(bufp, n, "\r\n\r\n", 4);

  162. 				m1 = memmem(bufp, n, "\n\n", 2);

  163. 				if (m0 && m1) {

  164. 					eoh = (m0 < m1 ? m0 + 4 : m1 + 2);

  165. 				} else if (m0) {

  166. 					eoh = m0 + 4;

  167. 				} else if (m1) {

  168. 					eoh = m1 + 2;

  169. 				} else {

  170. 					eoh = NULL;

  171. 				}

  172. 				if (!eoh) {

  173. 					fprintf(stderr, "Error: end of headers not found\n");

  174. 					return 1;

  175. 				}

  176. 				DEBUG_PRINTF("Response body begins at %p (offset %zu)\n", (void*)eoh, eoh - bufp);

  177. 

  178. 				if (memcmp(bufp, "HTTP/1.0 ", 9) == 0) {

  179. 					// HTTP 1.0 doesn't have TE, so just use the response_length for the content length and skip the other header parsing

  180. 					if (bufp + response_length < eoh) {

  181. 						fprintf(stderr, "Error: end of headers occurs after alleged response length\n");

  182. 						return 1;

  183. 					}

  184. 					length = response_length - (eoh - bufp);

  185. 					state = STATE_BODY;

  186. 				} else if ((m0 = memcasemem(bufp, eoh - bufp, "\ncontent-length:", 16)) && m0 < eoh) {

  187. 					DEBUG_PRINTF("Found Content-Length header at %p (offset %zu)\n", (void*)(m0 + 2), m0 + 2 - bufp);

  188. 					m1 = memmem(m0 + 1, eoh - (m0 + 1), "\n", 1);

  189. 					if (!m1) {

  190. 						fprintf(stderr, "Error: CRLF after Content-Length missing\n");

  191. 						return 1;

  192. 					}

  193. 					if (*(m1 - 1) == '\r') --m1;

  194. 					m0 += 16;

  195. 					while (m0 < bufp + n && (*m0 == ' ' || *m0 == '\t')) ++m0;

  196. 					if (sscanf(m0, "%zu%ln", &length, &nscan) <= 0) {

  197. 						fprintf(stderr, "Error: invalid Content-Length\n");

  198. 						return 1;

  199. 					}

  200. 					if (nscan < 0) {

  201. 						fprintf(stderr, "Error: negative nscan\n");

  202. 						return 1;

  203. 					}

  204. 					if (m0 + nscan > bufp + n) {

  205. 						fprintf(stderr, "Error: buffer overread\n");

  206. 						return 1;

  207. 					}

  208. 					m0 += nscan;

  209. 					while (m0 < bufp + n && (*m0 == ' ' || *m0 == '\t')) ++m0;

  210. 					if (m0 != m1) {

  211. 						fprintf(stderr, "Error: invalid Content-Length (noise before EOL)\n");

  212. 						return 1;

  213. 					}

  214. 					DEBUG_PRINTF("Content length: %zu\n", length);

  215. 

  216. 					state = STATE_BODY;

  217. 				} else {

  218. 					m0 = memcasemem(bufp, eoh - bufp, "\ntransfer-encoding:", 19);

  219. 					if (!m0 || m0 >= eoh) {

  220. 						DEBUG_PRINTF("No Content-Length or Transfer-Encoding, falling back to response length\n");

  221. 						if (!have_response_length) {

  222. 							fprintf(stderr, "Error: Content-Length and Transfer-Encoding missing and no response length from metadata line\n");

  223. 							return 1;

  224. 						}

  225. 						if (bufp + response_length < eoh) {

  226. 							fprintf(stderr, "Error: end of headers occurs after alleged response length\n");

  227. 							return 1;

  228. 						}

  229. 						length = response_length - (eoh - bufp);

  230. 						state = STATE_BODY;

  231. 					} else {

  232. 						DEBUG_PRINTF("Found Transfer-Encoding header at %p (offset %zu)\n", (void*)(m0 + 1), m0 + 1 - bufp);

  233. 						m1 = memmem(m0 + 1, eoh - (m0 + 1), "\n", 1);

  234. 						if (!m1 || m1 >= eoh - 1) {

  235. 							fprintf(stderr, "Error: CRLF after Transfer-Encoding missing\n");

  236. 							return 1;

  237. 						}

  238. 						m0 += 19;

  239. 						if (*(m1 - 1) == '\r') --m1;

  240. 						while (m0 < bufp + n && (*m0 == ' ' || *m0 == '\t')) ++m0;

  241. 						if (memcmp(m0, "chunked", 7) != 0) {

  242. 							fprintf(stderr, "Error: unsupported Transfer-Encoding\n");

  243. 							return 1;

  244. 						}

  245. 						m0 += 7;

  246. 						while (m0 < bufp + n && (*m0 == ' ' || *m0 == '\t')) ++m0;

  247. 						if (m0 != m1) {

  248. 							fprintf(stderr, "Error: unsupported Transfer-Encoding\n");

  249. 							return 1;

  250. 						}

  251. 						DEBUG_PRINTF("Chunked transfer encoding\n");

  252. 

  253. 						state = STATE_CHUNK_LINE;

  254. 					}

  255. 				}

  256. 

  257. 				if (html_fake_base) {

  258. 					m0 = memcasemem(bufp, eoh - bufp, "\ncontent-type:", 14);

  259. 					if (m0 && m0 < eoh) {

  260. 						DEBUG_PRINTF("Found Content-Type header at %p (offset %zu)\n", (void*)(m0 + 1), m0 + 1 - bufp);

  261. 						m1 = memmem(m0 + 1, eoh - (m0 + 1), "\n", 1);

  262. 						if (!m1) {

  263. 							fprintf(stderr, "Error: CRLF after Content-Type missing\n");

  264. 							return 1;

  265. 						}

  266. 						m0 += 14;

  267. 						while (m0 < bufp + n && (*m0 == ' ' || *m0 == '\t')) ++m0;

  268. 						if (memcmp(m0, "text/html", 9) == 0) {

  269. 							DEBUG_PRINTF("Is HTML response, inserting fake base tag\n");

  270. 							fprintf(stdout, "<base href=\"");

  271. 							fwrite(url, 1, urllen, stdout);

  272. 							fprintf(stdout, "\">\n");

  273. 						}

  274. 					}

  275. 				}

  276. 

  277. 				DEBUG_PRINTF("Adjusting buffer pointer and n by %zu\n", eoh - bufp);

  278. 				n = n - (eoh - bufp);

  279. 				bufp = eoh;

  280. 				bytes_read = 0;

  281. 				goto checkstate;

  282. 			} else {

  283. 				fprintf(stderr, "Error: expected header line, got something else\n");

  284. 				return 1;

  285. 			}

  286. 		} else if (state == STATE_BODY || state == STATE_CHUNK_CONTENTS) {

  287. 			if (length + (state == STATE_BODY ? 1 : 2) - bytes_read > n) {

  288. 				// Only got part of the body

  289. 				DEBUG_PRINTF("Partial body\n");

  290. 				DEBUG_PRINTF("Copying %zu bytes to stdout\n", n);

  291. 				fwrite(bufp, 1, n, stdout);

  292. 				bytes_read += n;

  293. 				DEBUG_PRINTF("%zu of %zu bytes from this response written\n", bytes_read, length);

  294. 			} else {

  295. 				// Remainder of the response is in the buffer. Same logic as above for small records fitting in the buffer with the headers.

  296. 				DEBUG_PRINTF("Full body\n");

  297. 				DEBUG_PRINTF("Copying %zu bytes to stdout\n", length - bytes_read);

  298. 				fwrite(bufp, 1, length - bytes_read, stdout);

  299. 				if (state == STATE_BODY) {

  300. 					fprintf(stdout, "\n");

  301. 				}

  302. 				if (state == STATE_CHUNK_CONTENTS && *(bufp + length - bytes_read) == '\r') {

  303. 					// Stupid hack to enforce the CRLF

  304. 					++length;

  305. 				}

  306. 				if (memcmp(bufp + length - bytes_read, "\n", 1) != 0) {

  307. 					fprintf(stderr, "Error: end of HTTP body not found\n");

  308. 					return 1;

  309. 				}

  310. 				DEBUG_PRINTF("Adjusting buffer pointer and n by %zu\n", length + 1 - bytes_read);

  311. 				n = n - (length + 1 - bytes_read);

  312. 				bufp = bufp + length + 1 - bytes_read;

  313. 				if (n < BUFSIZE) {

  314. 					DEBUG_PRINTF("Buffer too small (%zu bytes), moving and refilling\n", n);

  315. 					memmove(buf, bufp, n);

  316. 					bufp = buf;

  317. 					n += fread(buf + n, 1, BUFSIZE, stdin);

  318. 				}

  319. 				if (state == STATE_BODY) {

  320. 					state = STATE_HEADERS;

  321. 				} else {

  322. 					state = STATE_CHUNK_LINE;

  323. 				}

  324. 				goto checkstate;

  325. 			}

  326. 		} else if (state == STATE_CHUNK_LINE) {

  327. 			m1 = memmem(bufp, n, "\r\n", 2);

  328. 			if (!m1) {

  329. 				fprintf(stderr, "Error: chunk line EOL missing\n");

  330. 				return 1;

  331. 			}

  332. 			m0 = bufp;

  333. 			while (m0 < bufp + n && (*m0 == ' ' || *m0 == '\t')) ++m0;

  334. 			if (sscanf(m0, "%zx%ln", &length, &nscan) <= 0) {

  335. 				fprintf(stderr, "Error: invalid chunk length\n");

  336. 				return 1;

  337. 			}

  338. 			if (nscan < 0) {

  339. 				fprintf(stderr, "Error: negative nscan\n");

  340. 				return 1;

  341. 			}

  342. 			if (m0 + nscan > bufp + n) {

  343. 				fprintf(stderr, "Error: buffer overread\n");

  344. 				return 1;

  345. 			}

  346. 			m0 += nscan;

  347. 			while (m0 < m1 && (*m0 == ' ' || *m0 == '\t')) ++m0;

  348. 			if (*m0 != ';' && m0 != m1) {

  349. 				fprintf(stderr, "Error: invalid Content-Length (noise before EOL)\n");

  350. 				return 1;

  351. 			}

  352. 			DEBUG_PRINTF("Chunk length: %zu bytes\n", length);

  353. 

  354. 			DEBUG_PRINTF("Adjusting buffer pointer and n by %zu\n", m1 + 2 - bufp);

  355. 			n = n - (m1 + 2 - bufp);

  356. 			bufp = m1 + 2;

  357. 

  358. 			if (length == 0) {

  359. 				// End of response, must be followed by CRLF + LF

  360. 				if (n < 3) {

  361. 					fprintf(stderr, "Error: buffer exhausted while looking for empty chunk CRLF\n");

  362. 					return 1;

  363. 				}

  364. 				if (*(m1 + 2) != '\r' || *(m1 + 3) != '\n' || *(m1 + 4) != '\n') {

  365. 					fprintf(stderr, "Error: end of HTTP body not found\n");

  366. 					return 1;

  367. 				}

  368. 				fprintf(stdout, "\n");

  369. 				n -= 3;

  370. 				bufp += 3;

  371. 				state = STATE_HEADERS;

  372. 			} else {

  373. 				state = STATE_CHUNK_CONTENTS;

  374. 			}

  375. 

  376. 			if (n < BUFSIZE) {

  377. 				DEBUG_PRINTF("Buffer too small (%zu bytes), moving and refilling\n", n);

  378. 				memmove(buf, bufp, n);

  379. 				bufp = buf;

  380. 				n += fread(buf + n, 1, BUFSIZE, stdin);

  381. 			}

  382. 			goto checkstate;

  383. 		}

  384. 	}

  385. 	if (state != STATE_HEADERS) {

  386. 		fprintf(stderr, "Error: incomplete body at the end of input\n");

  387. 		return 1;

  388. 	}

  389. }