Geek Answers Handbook

How to handle data flow inside a C-based application?

I am extracting data from the bzip2 stream in application C. Since pieces of data come out of the decompressor, they can be written to stdout :

 fwrite(buffer, 1, length, stdout);

This works great. I get all the data when it is sent to stdout .

Instead of writing to stdout , I would like to process the output from this statement inside one row: a line that ends with a newline \n .

Do I write the output of the decompressor stream to another buffer, one character at a time, until I click on a new line, and then call the processing function in the line? Is it slow and a smarter approach? Thank you for your advice.

EDIT

Thanks for your suggestions. I ended up creating a pair of buffers that store the remainder (the "stub" at the end of the output buffer) at the beginning of the short string buffer every time I pass the value of the output buffer.

I process the character of the output buffer by the character and process the data in one line of a line at a time. The remainder without a new line is allocated and assigned and copied to the next stream line buffer. It appears that realloc less expensive than the repetitive malloc-free statements.

Here is the code I came up with:

 char bzBuf[BZBUFMAXLEN]; BZFILE *bzFp; int bzError, bzNBuf; char bzLineBuf[BZLINEBUFMAXLEN]; char *bzBufRemainder = NULL; int bzBufPosition, bzLineBufPosition; bzFp = BZ2_bzReadOpen(&bzError, *fp, 0, 0, NULL, 0); /* http://www.bzip.org/1.0.5/bzip2-manual-1.0.5.html#bzcompress-init */ if (bzError != BZ_OK) { BZ2_bzReadClose(&bzError, bzFp); fprintf(stderr, "\n\t[gchr2] - Error: Bzip2 data could not be retrieved\n\n"); return -1; } bzError = BZ_OK; bzLineBufPosition = 0; while (bzError == BZ_OK) { bzNBuf = BZ2_bzRead(&bzError, bzFp, bzBuf, sizeof(bzBuf)); if (bzError == BZ_OK || bzError == BZ_STREAM_END) { if (bzBufRemainder != NULL) { /* fprintf(stderr, "copying bzBufRemainder to bzLineBuf...\n"); */ strncpy(bzLineBuf, bzBufRemainder, strlen(bzBufRemainder)); /* leave out \0 */ bzLineBufPosition = strlen(bzBufRemainder); } for (bzBufPosition = 0; bzBufPosition < bzNBuf; bzBufPosition++) { bzLineBuf[bzLineBufPosition++] = bzBuf[bzBufPosition]; if (bzBuf[bzBufPosition] == '\n') { bzLineBuf[bzLineBufPosition] = '\0'; /* terminate bzLineBuf */ /* process the line buffer, eg print it out or transform it, etc. */ fprintf(stdout, "%s", bzLineBuf); bzLineBufPosition = 0; /* reset line buffer position */ } else if (bzBufPosition == (bzNBuf - 1)) { bzLineBuf[bzLineBufPosition] = '\0'; if (bzBufRemainder != NULL) bzBufRemainder = (char *)realloc(bzBufRemainder, bzLineBufPosition); else bzBufRemainder = (char *)malloc(bzLineBufPosition); strncpy(bzBufRemainder, bzLineBuf, bzLineBufPosition); } } } } if (bzError != BZ_STREAM_END) { BZ2_bzReadClose(&bzError, bzFp); fprintf(stderr, "\n\t[gchr2] - Error: Bzip2 data could not be uncompressed\n\n"); return -1; } else { BZ2_bzReadGetUnused(&bzError, bzFp, 0, 0); BZ2_bzReadClose(&bzError, bzFp); } free(bzBufRemainder); bzBufRemainder = NULL;

I am very grateful to everyone for their help. It works great.

+4
source share
4 answers

This would be easy to do using C ++ std::string , but C requires some code if you want to do it efficiently (unless you use a dynamic string library).

 char *bz_read_line(BZFILE *input) { size_t offset = 0; size_t len = CHUNK; // arbitrary char *output = (char *)xmalloc(len); int bzerror; while (BZ2_bzRead(&bzerror, input, output + offset, 1) == 1) { if (offset+1 == len) { len += CHUNK; output = xrealloc(output, len); } if (output[offset] == '\n') break; offset++; } if (output[offset] == '\n') output[offset] = '\0'; // strip trailing newline else if (bzerror != BZ_STREAM_END) { free(output); return NULL; } return output; }

(Where xmalloc and xrealloc handle internal errors. Remember to free return the string.)

This is almost an order of magnitude slower than bzcat :

 lars@zygmunt :/tmp$ wc foo 1193 5841 42868 foo lars@zygmunt :/tmp$ bzip2 foo lars@zygmunt :/tmp$ time bzcat foo.bz2 > /dev/null real 0m0.010s user 0m0.008s sys 0m0.000s lars@zygmunt :/tmp$ time ./a.out < foo.bz2 > /dev/null real 0m0.093s user 0m0.044s sys 0m0.020s

Determine how acceptable this is.

+1
source

I don’t think that there is a more reasonable approach (besides finding a library of automata that already does this for you). Be careful about assigning the correct size for the "last line" buffer: if it cannot handle an arbitrary length, and the input comes from something available to third parties, this becomes a security risk.

+2
source

I also worked with bzip2 data processing per line, and I found that reading one byte at a time was too slow. This helped me better:

 #include <stdio.h> #include <stdlib.h> #include <bzlib.h> /* gcc -o bz bz.c -lbz2 */ #define CHUNK 128 struct bzdata { FILE *fp; BZFILE *bzf; int bzeof, bzlen, bzpos; char bzbuf[4096]; }; static int bz2_open(struct bzdata *bz, char *file); static void bz2_close(struct bzdata *bz); static int bz2_read_line(struct bzdata *bz, char **line, int *li); static int bz2_buf(struct bzdata *bz, char **line, int *li, int *ll); static int bz2_buf(struct bzdata *bz, char **line, int *li, int *ll) { int done = 0; for (; bz->bzpos < bz->bzlen && done == 0; bz->bzpos++) { if (*ll + 1 >= *li) { *li += CHUNK; *line = realloc(*line, (*li + 1) * sizeof(*(*line))); } if ( ((*line)[(*ll)++] = bz->bzbuf[bz->bzpos]) == '\n') { done = 1; } } if (bz->bzpos == bz->bzlen) { bz->bzpos = bz->bzlen = 0; } (*line)[*ll] = '\0'; return done; } static int bz2_read_line(struct bzdata *bz, char **line, int *li) { int bzerr = BZ_OK, done = 0, ll = 0; if (bz->bzpos) { done = bz2_buf(bz, line, li, &ll); } while (done == 0 && bz->bzeof == 0) { bz->bzlen = BZ2_bzRead(&bzerr, bz->bzf, bz->bzbuf, sizeof(bz->bzbuf)); if (bzerr == BZ_OK || bzerr == BZ_STREAM_END) { bz->bzpos = 0; if (bzerr == BZ_STREAM_END) { bz->bzeof = 1; } done = bz2_buf(bz, line, li, &ll); } else { done = -1; } } /* Handle last lines that don't have a line feed */ if (done == 0 && ll > 0 && bz->bzeof) { done = 1; } return done; } static int bz2_open(struct bzdata *bz, char *file) { int bzerr = BZ_OK; if ( (bz->fp = fopen(file, "rb")) && (bz->bzf = BZ2_bzReadOpen(&bzerr, bz->fp, 0, 0, NULL, 0)) && bzerr == BZ_OK) { return 1; } return 0; } static void bz2_close(struct bzdata *bz) { int bzerr; if (bz->bzf) { BZ2_bzReadClose(&bzerr, bz->bzf); bz->bzf = NULL; } if (bz->fp) { fclose(bz->fp); bz->fp = NULL; } bz->bzpos = bz->bzlen = bz->bzeof = 0; } int main(int argc, char *argv[]) { struct bzdata *bz = NULL; int i, lc, li = 0; char *line = NULL; if (argc < 2) { return fprintf(stderr, "usage: %s file [file ...]\n", argv[0]); } if ( (bz = calloc(1, sizeof(*bz))) ) { for (i = 1; i < argc; i++) { if (bz2_open(bz, argv[i])) { for (lc = 0; bz2_read_line(bz, &line, &li) > 0; lc++) { /* Process line here */ } printf("%s: lines=%d\n", argv[i], lc); } bz2_close(bz); } free(bz); } if (line) { free(line); } return 0; }
+2
source

I think you need to copy the chunks of characters to another buffer until the last chunk you write contains a new line character. Then you can work on the entire line.

You can save the rest of the buffer (after '\n' ) into a temporary one, and then create a new line from it.

0
source

More articles:


All Articles