2 tofuproxy -- flexible HTTP proxy, TLS terminator, X.509 certificates
3 manager, WARC/Gemini browser
4 Copyright (C) 2021 Sergey Matveev <stargrave@stargrave.org>
6 This program is free software: you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation, version 3 of the License.
10 This program is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 GNU General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
19 // https://iipc.github.io/warc-specifications/specifications/warc-zstd/
27 #include <sys/endian.h>
32 main(int argc, char **argv)
34 ZSTD_DCtx *ctx = ZSTD_createDCtx();
36 fputs("can not initialize ZSTD_DCtx\n", stderr);
39 FILE *fdOff = fdopen(3, "wb");
40 int rc = EXIT_FAILURE;
41 uint8_t *bufIn = NULL;
42 uint8_t *bufOut = NULL;
43 const size_t bufInSize = ZSTD_DStreamInSize();
44 bufIn = malloc(bufInSize);
46 fputs("no memory\n", stderr);
49 const size_t bufOutSize = ZSTD_DStreamOutSize();
50 bufOut = malloc(bufOutSize);
52 fputs("no memory\n", stderr);
56 unsigned long long bufSize = 0;
58 ZSTD_inBuffer bIn = {bufIn, 0, 0};
59 ZSTD_outBuffer bOut = {bufOut, 0, 0};
62 bool lastBlock = false;
66 size_t offsetPrev = 0;
70 n = fread(bufIn, 1, bufInSize, stdin);
75 perror("can not fread(FILE)");
79 if (n >= 8 && le32dec(bufIn) == 0x184D2A5D) {
81 size_t dictSize = (size_t)le32dec(bufIn + 4);
82 uint8_t *dict = malloc(dictSize);
84 fprintf(stderr, "insufficient memory for dictionary: %zu\n", dictSize);
87 const size_t alreadyRead = n - 8;
88 memcpy(dict, bufIn + 8, alreadyRead);
90 n = fread(dict + alreadyRead, 1, dictSize - alreadyRead, stdin);
91 if (n != dictSize - alreadyRead) {
92 perror("can not read dictionary data");
96 offset = dictSize + 8;
99 fprintf(fdOff, "%zu\t0\n", offset);
101 uint32_t hdr = le32dec(dict);
103 case ZSTD_MAGIC_DICTIONARY:
104 zCode = ZSTD_DCtx_loadDictionary(ctx, dict, dictSize);
106 if ((zCode != 0) && (ZSTD_isError(zCode))) {
109 "can not load dictionary: %s\n",
110 ZSTD_getErrorName(zCode));
115 case ZSTD_MAGICNUMBER:
116 bufSize = ZSTD_getFrameContentSize(dict, dictSize);
118 case ZSTD_CONTENTSIZE_UNKNOWN:
119 case ZSTD_CONTENTSIZE_ERROR:
120 fprintf(stderr, "can not determine dictionary's size\n");
124 uint8_t *buf = malloc(bufSize);
127 stderr, "insufficient memory for dictionary: %llu\n", bufSize);
131 zCode = ZSTD_decompress(buf, bufSize, dict, dictSize);
133 if (ZSTD_isError(zCode)) {
136 "can not decompress dictionary: %s\n",
137 ZSTD_getErrorName(zCode));
141 zCode = ZSTD_DCtx_loadDictionary(ctx, buf, zCode);
143 if ((zCode != 0) && (ZSTD_isError(zCode))) {
146 "can not load dictionary: %s\n",
147 ZSTD_getErrorName(zCode));
153 fprintf(stderr, "unknown dictionary header\n");
161 while (bIn.pos < bIn.size) {
162 bOut.size = bufOutSize;
164 zCode = ZSTD_decompressStream(ctx, &bOut, &bIn);
165 if ((zCode != 0) && (ZSTD_isError(zCode))) {
166 fprintf(stderr, "can not decompress: %s\n", ZSTD_getErrorName(zCode));
169 n = fwrite(bufOut, 1, bOut.pos, stdout);
171 perror("can not fwrite(stdout)");
178 fprintf(fdOff, "%zu\t%zu\n", offset - offsetPrev, written);
180 offsetPrev = offset + bIn.pos;
191 fputs("empty input\n", stderr);
195 fprintf(stderr, "unfinished decompression: %s\n", ZSTD_getErrorName(zCode));
201 if (bufOut != NULL) {
208 if ((fdOff != NULL) && (fclose(fdOff) != 0)) {
209 perror("can not fclose(4)");