![[BACK]](/icons/back.gif){kind=icon}
Return to file.c CVS log ![[TXT]](/icons/text.gif){kind=icon}
![[DIR]](/icons/dir.gif){kind=icon}
Up to [local] / src / usr.bin / sort|
Return to file.c CVS log | Up to [local] / src / usr.bin / sort |
1.19 ! deraadt 1: /* $OpenBSD: file.c,v 1.18 2015/04/02 21:04:06 tobias Exp $ */
1.1 millert 2:
3: /*-
4: * Copyright (C) 2009 Gabor Kovesdan <gabor@FreeBSD.org>
5: * Copyright (C) 2012 Oleg Moskalenko <mom040267@gmail.com>
6: * All rights reserved.
7: *
8: * Redistribution and use in source and binary forms, with or without
9: * modification, are permitted provided that the following conditions
10: * are met:
11: * 1. Redistributions of source code must retain the above copyright
12: * notice, this list of conditions and the following disclaimer.
13: * 2. Redistributions in binary form must reproduce the above copyright
14: * notice, this list of conditions and the following disclaimer in the
15: * documentation and/or other materials provided with the distribution.
16: *
17: * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18: * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19: * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20: * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21: * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22: * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23: * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24: * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25: * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26: * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27: * SUCH DAMAGE.
28: */
29:
30: #include <sys/mman.h>
31: #include <sys/stat.h>
32: #include <sys/types.h>
33: #include <sys/queue.h>
34:
35: #include <err.h>
36: #include <fcntl.h>
37: #include <stdio.h>
38: #include <stdlib.h>
1.14 deraadt 39: #include <signal.h>
1.1 millert 40: #include <string.h>
41: #include <unistd.h>
42: #include <wchar.h>
43: #include <wctype.h>
44:
45: #include "coll.h"
46: #include "file.h"
47: #include "radixsort.h"
48:
49: unsigned long long available_free_memory = 1000000;
50:
51: bool use_mmap;
52:
53: const char *tmpdir = "/var/tmp";
54: const char *compress_program;
55:
56: size_t max_open_files = 16;
57:
58: /*
59: * How much space we read from file at once
60: */
61: #define READ_CHUNK 4096
62:
63: /*
64: * File reader structure
65: */
66: struct file_reader {
67: struct reader_buffer rb;
68: FILE *file;
69: char *fname;
70: unsigned char *buffer;
71: unsigned char *mmapaddr;
72: unsigned char *mmapptr;
73: size_t bsz;
74: size_t cbsz;
75: size_t mmapsize;
76: size_t strbeg;
77: char elsymb;
78: };
79:
80: /*
81: * Structure to be used in file merge process.
82: */
83: struct file_header {
84: struct file_reader *fr;
85: struct sort_list_item *si; /* current top line */
86: size_t file_pos;
87: };
88:
89: /*
90: * List elements of "cleanable" files list.
91: */
92: struct CLEANABLE_FILE {
93: char *fn;
94: LIST_ENTRY(CLEANABLE_FILE) files;
95: };
96:
97: /*
98: * List header of "cleanable" files list.
99: */
100: static LIST_HEAD(CLEANABLE_FILES, CLEANABLE_FILE) tmp_files;
101:
102: /*
103: * Init tmp files list
104: */
105: void
106: init_tmp_files(void)
107: {
108: LIST_INIT(&tmp_files);
109: }
110:
111: /*
112: * Save name of a tmp file for signal cleanup
113: */
114: void
115: tmp_file_atexit(const char *tmp_file)
116: {
1.12 millert 117: struct CLEANABLE_FILE *item;
1.14 deraadt 118: sigset_t mask, oldmask;
1.12 millert 119:
120: item = sort_malloc(sizeof(struct CLEANABLE_FILE));
121: item->fn = sort_strdup(tmp_file);
1.14 deraadt 122:
123: sigfillset(&mask);
124: sigprocmask(SIG_BLOCK, &mask, &oldmask);
1.12 millert 125: LIST_INSERT_HEAD(&tmp_files, item, files);
1.14 deraadt 126: sigprocmask(SIG_SETMASK, &oldmask, NULL);
1.1 millert 127: }
128:
129: /*
130: * Clear tmp files
131: */
132: void
133: clear_tmp_files(void)
134: {
135: struct CLEANABLE_FILE *item;
136:
137: LIST_FOREACH(item, &tmp_files, files) {
138: if (item != NULL && item->fn != NULL)
139: unlink(item->fn);
140: }
141: }
142:
143: /*
144: * Check whether a file is a temporary file
145: */
146: static bool
147: file_is_tmp(const char *fn)
148: {
149: struct CLEANABLE_FILE *item;
150:
1.12 millert 151: LIST_FOREACH(item, &tmp_files, files) {
152: if (item->fn != NULL && strcmp(item->fn, fn) == 0)
153: return true;
1.1 millert 154: }
155:
1.12 millert 156: return false;
1.1 millert 157: }
158:
159: /*
160: * Generate new temporary file name
161: */
162: char *
163: new_tmp_file_name(void)
164: {
165: char *ret;
1.7 tobias 166: int fd;
1.1 millert 167:
1.7 tobias 168: sort_asprintf(&ret, "%s/.bsdsort.XXXXXXXXXX", tmpdir);
169: if ((fd = mkstemp(ret)) == -1)
170: err(2, "%s", ret);
171: close(fd);
1.1 millert 172: tmp_file_atexit(ret);
173: return ret;
174: }
175:
176: /*
177: * Initialize file list
178: */
179: void
180: file_list_init(struct file_list *fl, bool tmp)
181: {
1.12 millert 182: fl->count = 0;
183: fl->sz = 0;
184: fl->fns = NULL;
185: fl->tmp = tmp;
1.1 millert 186: }
187:
188: /*
189: * Add a file name to the list
190: */
191: void
192: file_list_add(struct file_list *fl, char *fn, bool allocate)
193: {
1.12 millert 194: if (fl->count >= fl->sz) {
195: fl->fns = sort_reallocarray(fl->fns,
196: fl->sz ? fl->sz : (fl->sz = 1), 2 * sizeof(char *));
197: fl->sz *= 2;
1.1 millert 198: }
1.12 millert 199: fl->fns[fl->count] = allocate ? sort_strdup(fn) : fn;
200: fl->count += 1;
1.1 millert 201: }
202:
203: /*
204: * Populate file list from array of file names
205: */
206: void
207: file_list_populate(struct file_list *fl, int argc, char **argv, bool allocate)
208: {
1.12 millert 209: int i;
1.1 millert 210:
1.12 millert 211: for (i = 0; i < argc; i++)
212: file_list_add(fl, argv[i], allocate);
1.1 millert 213: }
214:
215: /*
216: * Clean file list data and delete the files,
217: * if this is a list of temporary files
218: */
219: void
220: file_list_clean(struct file_list *fl)
221: {
1.12 millert 222: if (fl->fns) {
223: size_t i;
1.1 millert 224:
1.12 millert 225: for (i = 0; i < fl->count; i++) {
226: if (fl->fns[i]) {
227: if (fl->tmp)
228: unlink(fl->fns[i]);
229: sort_free(fl->fns[i]);
230: fl->fns[i] = NULL;
1.1 millert 231: }
232: }
1.12 millert 233: sort_free(fl->fns);
234: fl->fns = NULL;
1.1 millert 235: }
1.12 millert 236: fl->sz = 0;
237: fl->count = 0;
238: fl->tmp = false;
1.1 millert 239: }
240:
241: /*
242: * Init sort list
243: */
244: void
245: sort_list_init(struct sort_list *l)
246: {
1.12 millert 247: l->count = 0;
248: l->size = 0;
249: l->memsize = sizeof(struct sort_list);
250: l->list = NULL;
1.1 millert 251: }
252:
253: /*
254: * Add string to sort list
255: */
256: void
257: sort_list_add(struct sort_list *l, struct bwstring *str)
258: {
1.12 millert 259: size_t indx = l->count;
1.1 millert 260:
1.12 millert 261: if ((l->list == NULL) || (indx >= l->size)) {
262: size_t newsize = (l->size + 1) + 1024;
1.1 millert 263:
1.12 millert 264: l->list = sort_reallocarray(l->list, newsize,
265: sizeof(struct sort_list_item *));
266: l->memsize += (newsize - l->size) *
267: sizeof(struct sort_list_item *);
268: l->size = newsize;
269: }
270: l->list[indx] = sort_list_item_alloc();
271: sort_list_item_set(l->list[indx], str);
272: l->memsize += sort_list_item_size(l->list[indx]);
273: l->count += 1;
1.1 millert 274: }
275:
276: /*
277: * Clean sort list data
278: */
279: void
280: sort_list_clean(struct sort_list *l)
281: {
1.12 millert 282: if (l->list) {
283: size_t i;
1.1 millert 284:
1.12 millert 285: for (i = 0; i < l->count; i++) {
286: struct sort_list_item *item;
1.1 millert 287:
1.12 millert 288: item = l->list[i];
1.1 millert 289:
1.12 millert 290: if (item) {
291: sort_list_item_clean(item);
292: sort_free(item);
293: l->list[i] = NULL;
1.1 millert 294: }
295: }
1.12 millert 296: sort_free(l->list);
297: l->list = NULL;
1.1 millert 298: }
1.12 millert 299: l->count = 0;
300: l->size = 0;
301: l->memsize = sizeof(struct sort_list);
1.1 millert 302: }
303:
304: /*
305: * Write sort list to file
306: */
307: void
308: sort_list_dump(struct sort_list *l, const char *fn)
309: {
1.12 millert 310: FILE *f;
311:
312: f = openfile(fn, "w");
313: if (f == NULL)
314: err(2, "%s", fn);
1.1 millert 315:
1.12 millert 316: if (l->list) {
317: size_t i;
1.1 millert 318:
1.12 millert 319: if (!sort_opts_vals.uflag) {
320: for (i = 0; i < l->count; ++i)
321: bwsfwrite(l->list[i]->str, f,
322: sort_opts_vals.zflag);
323: } else {
324: struct sort_list_item *last_printed_item = NULL;
325: struct sort_list_item *item;
326: for (i = 0; i < l->count; ++i) {
327: item = l->list[i];
328: if ((last_printed_item == NULL) ||
329: list_coll(&last_printed_item, &item)) {
330: bwsfwrite(item->str, f, sort_opts_vals.zflag);
331: last_printed_item = item;
1.1 millert 332: }
333: }
334: }
1.12 millert 335: }
1.1 millert 336:
1.12 millert 337: closefile(f, fn);
1.1 millert 338: }
339:
340: /*
341: * Checks if the given file is sorted. Stops at the first disorder,
342: * prints the disordered line and returns 1.
343: */
344: int
345: check(const char *fn)
346: {
347: struct bwstring *s1, *s2, *s1disorder, *s2disorder;
348: struct file_reader *fr;
349: struct keys_array *ka1, *ka2;
350: int res;
351: size_t pos, posdisorder;
352:
353: s1 = s2 = s1disorder = s2disorder = NULL;
354: ka1 = ka2 = NULL;
355:
356: fr = file_reader_init(fn);
357:
358: res = 0;
359: pos = 1;
360: posdisorder = 1;
361:
362: if (fr == NULL) {
363: err(2, "%s", fn);
364: goto end;
365: }
366:
367: s1 = file_reader_readline(fr);
368: if (s1 == NULL)
369: goto end;
370:
371: ka1 = keys_array_alloc();
372: preproc(s1, ka1);
373:
374: s2 = file_reader_readline(fr);
375: if (s2 == NULL)
376: goto end;
377:
378: ka2 = keys_array_alloc();
379: preproc(s2, ka2);
380:
381: for (;;) {
382:
383: if (debug_sort) {
384: bwsprintf(stdout, s2, "s1=<", ">");
385: bwsprintf(stdout, s1, "s2=<", ">");
386: }
387: int cmp = key_coll(ka2, ka1, 0);
388: if (debug_sort)
389: printf("; cmp1=%d", cmp);
390:
391: if (!cmp && sort_opts_vals.complex_sort &&
392: !(sort_opts_vals.uflag) && !(sort_opts_vals.sflag)) {
393: cmp = top_level_str_coll(s2, s1);
394: if (debug_sort)
395: printf("; cmp2=%d", cmp);
396: }
397: if (debug_sort)
398: printf("\n");
399:
400: if ((sort_opts_vals.uflag && (cmp <= 0)) || (cmp < 0)) {
401: if (!(sort_opts_vals.csilentflag)) {
402: s2disorder = bwsdup(s2);
403: posdisorder = pos;
404: if (debug_sort)
405: s1disorder = bwsdup(s1);
406: }
407: res = 1;
408: goto end;
409: }
410:
411: pos++;
412:
413: clean_keys_array(s1, ka1);
414: sort_free(ka1);
415: ka1 = ka2;
416: ka2 = NULL;
417:
418: bwsfree(s1);
419: s1 = s2;
420:
421: s2 = file_reader_readline(fr);
422: if (s2 == NULL)
423: goto end;
424:
425: ka2 = keys_array_alloc();
426: preproc(s2, ka2);
427: }
428:
429: end:
430: if (ka1) {
431: clean_keys_array(s1, ka1);
432: sort_free(ka1);
433: }
434:
435: if (s1)
436: bwsfree(s1);
437:
438: if (ka2) {
439: clean_keys_array(s2, ka2);
440: sort_free(ka2);
441: }
442:
443: if (s2)
444: bwsfree(s2);
445:
446: if (fn == NULL || *fn == 0 || strcmp(fn, "-") == 0) {
447: for (;;) {
448: s2 = file_reader_readline(fr);
449: if (s2 == NULL)
450: break;
451: bwsfree(s2);
452: }
453: }
454:
455: file_reader_free(fr);
456:
457: if (s2disorder) {
458: bws_disorder_warnx(s2disorder, fn, posdisorder);
459: if (s1disorder) {
460: bws_disorder_warnx(s1disorder, fn, posdisorder);
461: if (s1disorder != s2disorder)
462: bwsfree(s1disorder);
463: }
464: bwsfree(s2disorder);
465: s1disorder = NULL;
466: s2disorder = NULL;
467: }
468:
469: if (res)
470: exit(res);
471:
472: return 0;
473: }
474:
475: /*
476: * Opens a file. If the given filename is "-", stdout will be
477: * opened.
478: */
479: FILE *
480: openfile(const char *fn, const char *mode)
481: {
482: FILE *file;
483:
1.17 millert 484: if (strcmp(fn, "-") == 0)
485: return (mode[0] == 'r') ? stdin : stdout;
1.1 millert 486:
1.17 millert 487: if (file_is_tmp(fn) && (compress_program != NULL)) {
488: char *cmd;
1.1 millert 489:
1.17 millert 490: fflush(stdout);
1.1 millert 491:
1.17 millert 492: if (mode[0] == 'r')
493: sort_asprintf(&cmd, "%s -d < %s",
494: compress_program, fn);
495: else if (mode[0] == 'w')
496: sort_asprintf(&cmd, "%s > %s",
497: compress_program, fn);
498: else
499: err(2, "invalid file mode");
500:
501: if ((file = popen(cmd, mode)) == NULL)
502: err(2, "%s", compress_program);
503:
504: sort_free(cmd);
505:
506: } else if ((file = fopen(fn, mode)) == NULL)
507: err(2, "%s", fn);
1.1 millert 508:
509: return file;
510: }
511:
512: /*
513: * Close file
514: */
515: void
516: closefile(FILE *f, const char *fn)
517: {
518: if (f == NULL) {
519: ;
520: } else if (f == stdin) {
521: ;
522: } else if (f == stdout) {
523: fflush(f);
524: } else {
525: if (file_is_tmp(fn) && compress_program != NULL) {
526: if (pclose(f) < 0)
527: err(2, NULL);
528: } else
529: fclose(f);
530: }
531: }
532:
533: /*
534: * Reads a file into the internal buffer.
535: */
536: struct file_reader *
537: file_reader_init(const char *fsrc)
538: {
539: struct file_reader *ret;
540:
541: if (fsrc == NULL)
542: fsrc = "-";
543:
1.2 millert 544: ret = sort_calloc(1, sizeof(struct file_reader));
1.1 millert 545:
546: ret->elsymb = '\n';
547: if (sort_opts_vals.zflag)
548: ret->elsymb = 0;
549:
550: ret->fname = sort_strdup(fsrc);
551:
552: if (strcmp(fsrc, "-") && (compress_program == NULL) && use_mmap) {
553: struct stat stat_buf;
554: void *addr;
555: size_t sz = 0;
556: int fd;
557:
558: fd = open(fsrc, O_RDONLY);
559: if (fd < 0)
560: err(2, "%s", fsrc);
561:
562: if (fstat(fd, &stat_buf) < 0)
563: err(2, "%s", fsrc);
564: sz = stat_buf.st_size;
565:
566: addr = mmap(NULL, sz, PROT_READ, 0, fd, 0);
1.16 millert 567: close(fd);
568: if (addr != MAP_FAILED) {
1.1 millert 569: ret->mmapaddr = addr;
570: ret->mmapsize = sz;
571: ret->mmapptr = ret->mmapaddr;
572: posix_madvise(addr, sz, POSIX_MADV_SEQUENTIAL);
573: }
574: }
575:
576: if (ret->mmapaddr == NULL) {
577: ret->file = openfile(fsrc, "r");
578: if (ret->file == NULL)
579: err(2, "%s", fsrc);
580:
581: if (strcmp(fsrc, "-")) {
582: ret->cbsz = READ_CHUNK;
583: ret->buffer = sort_malloc(ret->cbsz);
584: ret->bsz = 0;
585: ret->strbeg = 0;
586:
587: ret->bsz = fread(ret->buffer, 1, ret->cbsz, ret->file);
588: if (ret->bsz == 0) {
589: if (ferror(ret->file))
590: err(2, NULL);
591: }
592: }
593: }
594:
595: return ret;
596: }
597:
598: struct bwstring *
599: file_reader_readline(struct file_reader *fr)
600: {
601: struct bwstring *ret = NULL;
602:
603: if (fr->mmapaddr) {
604: unsigned char *mmapend;
605:
606: mmapend = fr->mmapaddr + fr->mmapsize;
607: if (fr->mmapptr >= mmapend)
608: return NULL;
609: else {
610: unsigned char *strend;
611: size_t sz;
612:
613: sz = mmapend - fr->mmapptr;
614: strend = memchr(fr->mmapptr, fr->elsymb, sz);
615:
616: if (strend == NULL) {
617: ret = bwscsbdup(fr->mmapptr, sz);
618: fr->mmapptr = mmapend;
619: } else {
620: ret = bwscsbdup(fr->mmapptr, strend -
621: fr->mmapptr);
622: fr->mmapptr = strend + 1;
623: }
624: }
625:
626: } else if (fr->file != stdin) {
627: unsigned char *strend;
628: size_t bsz1, remsz, search_start;
629:
630: search_start = 0;
631: remsz = 0;
632: strend = NULL;
633:
634: if (fr->bsz > fr->strbeg)
635: remsz = fr->bsz - fr->strbeg;
636:
637: /* line read cycle */
638: for (;;) {
639: if (remsz > search_start)
640: strend = memchr(fr->buffer + fr->strbeg +
641: search_start, fr->elsymb, remsz -
642: search_start);
643: else
644: strend = NULL;
645:
646: if (strend)
647: break;
648: if (feof(fr->file))
649: break;
650:
651: if (fr->bsz != fr->cbsz)
652: /* NOTREACHED */
653: err(2, "File read software error 1");
654:
655: if (remsz > (READ_CHUNK >> 1)) {
656: search_start = fr->cbsz - fr->strbeg;
657: fr->cbsz += READ_CHUNK;
1.13 millert 658: fr->buffer = sort_reallocarray(fr->buffer,
659: 1, fr->cbsz);
1.1 millert 660: bsz1 = fread(fr->buffer + fr->bsz, 1,
661: READ_CHUNK, fr->file);
662: if (bsz1 == 0) {
663: if (ferror(fr->file))
664: err(2, NULL);
665: break;
666: }
667: fr->bsz += bsz1;
668: remsz += bsz1;
669: } else {
1.12 millert 670: if (remsz > 0 && fr->strbeg > 0) {
671: memmove(fr->buffer,
672: fr->buffer + fr->strbeg, remsz);
673: }
1.1 millert 674: fr->strbeg = 0;
675: search_start = remsz;
676: bsz1 = fread(fr->buffer + remsz, 1,
677: fr->cbsz - remsz, fr->file);
678: if (bsz1 == 0) {
679: if (ferror(fr->file))
680: err(2, NULL);
681: break;
682: }
683: fr->bsz = remsz + bsz1;
684: remsz = fr->bsz;
685: }
686: }
687:
688: if (strend == NULL)
689: strend = fr->buffer + fr->bsz;
690:
691: if ((fr->buffer + fr->strbeg <= strend) &&
692: (fr->strbeg < fr->bsz) && (remsz>0))
693: ret = bwscsbdup(fr->buffer + fr->strbeg, strend -
694: fr->buffer - fr->strbeg);
695:
696: fr->strbeg = (strend - fr->buffer) + 1;
697: } else {
698: size_t len = 0;
699:
700: ret = bwsfgetln(fr->file, &len, sort_opts_vals.zflag,
701: &(fr->rb));
702: }
703:
704: return ret;
705: }
706:
707: static void
708: file_reader_clean(struct file_reader *fr)
709: {
1.12 millert 710: if (fr->mmapaddr)
711: munmap(fr->mmapaddr, fr->mmapsize);
1.1 millert 712:
1.12 millert 713: sort_free(fr->buffer);
1.1 millert 714:
1.12 millert 715: if (fr->file)
1.15 millert 716: closefile(fr->file, fr->fname);
1.1 millert 717:
1.12 millert 718: sort_free(fr->fname);
1.1 millert 719:
1.12 millert 720: memset(fr, 0, sizeof(struct file_reader));
1.1 millert 721: }
722:
723: void
724: file_reader_free(struct file_reader *fr)
725: {
1.12 millert 726: file_reader_clean(fr);
727: sort_free(fr);
1.1 millert 728: }
729:
730: int
731: procfile(const char *fsrc, struct sort_list *list, struct file_list *fl)
732: {
733: struct file_reader *fr;
734:
735: fr = file_reader_init(fsrc);
736: if (fr == NULL)
737: err(2, "%s", fsrc);
738:
739: /* file browse cycle */
740: for (;;) {
741: struct bwstring *bws;
742:
743: bws = file_reader_readline(fr);
744:
745: if (bws == NULL)
746: break;
747:
748: sort_list_add(list, bws);
749:
750: if (list->memsize >= available_free_memory) {
751: char *fn;
752:
753: fn = new_tmp_file_name();
754: sort_list_to_file(list, fn);
755: file_list_add(fl, fn, false);
756: sort_list_clean(list);
757: }
758: }
759:
760: file_reader_free(fr);
761:
762: return 0;
763: }
764:
765: /*
766: * Compare file headers. Files with EOF always go to the end of the list.
767: */
768: static int
769: file_header_cmp(struct file_header *f1, struct file_header *f2)
770: {
1.12 millert 771: int ret;
772:
1.1 millert 773: if (f1 == f2)
774: return 0;
1.12 millert 775: if (f1->fr == NULL)
776: return (f2->fr == NULL) ? 0 : 1;
777: if (f2->fr == NULL)
778: return -1;
779:
780: ret = list_coll(&(f1->si), &(f2->si));
781: if (!ret)
782: return (f1->file_pos < f2->file_pos) ? -1 : 1;
783: return ret;
1.1 millert 784: }
785:
786: /*
787: * Allocate and init file header structure
788: */
789: static void
790: file_header_init(struct file_header **fh, const char *fn, size_t file_pos)
791: {
1.12 millert 792: struct bwstring *line;
1.1 millert 793:
1.12 millert 794: *fh = sort_malloc(sizeof(struct file_header));
795: (*fh)->file_pos = file_pos;
796: (*fh)->fr = file_reader_init(fn);
797: if ((*fh)->fr == NULL) {
798: err(2, "Cannot open %s for reading",
799: strcmp(fn, "-") == 0 ? "stdin" : fn);
800: }
801: line = file_reader_readline((*fh)->fr);
802: if (line == NULL) {
803: file_reader_free((*fh)->fr);
804: (*fh)->fr = NULL;
805: (*fh)->si = NULL;
806: } else {
807: (*fh)->si = sort_list_item_alloc();
808: sort_list_item_set((*fh)->si, line);
1.1 millert 809: }
810: }
811:
812: /*
813: * Close file
814: */
815: static void
816: file_header_close(struct file_header **fh)
817: {
1.12 millert 818: if ((*fh)->fr) {
819: file_reader_free((*fh)->fr);
820: (*fh)->fr = NULL;
821: }
822: if ((*fh)->si) {
823: sort_list_item_clean((*fh)->si);
824: sort_free((*fh)->si);
825: (*fh)->si = NULL;
1.1 millert 826: }
1.12 millert 827: sort_free(*fh);
828: *fh = NULL;
1.1 millert 829: }
830:
831: /*
832: * Swap two array elements
833: */
834: static void
835: file_header_swap(struct file_header **fh, size_t i1, size_t i2)
836: {
837: struct file_header *tmp;
838:
839: tmp = fh[i1];
840: fh[i1] = fh[i2];
841: fh[i2] = tmp;
842: }
843:
844: /* heap algorithm ==>> */
845:
846: /*
847: * See heap sort algorithm
848: * "Raises" last element to its right place
849: */
850: static void
851: file_header_heap_swim(struct file_header **fh, size_t indx)
852: {
853: if (indx > 0) {
854: size_t parent_index;
855:
856: parent_index = (indx - 1) >> 1;
857:
858: if (file_header_cmp(fh[indx], fh[parent_index]) < 0) {
859: /* swap child and parent and continue */
860: file_header_swap(fh, indx, parent_index);
861: file_header_heap_swim(fh, parent_index);
862: }
863: }
864: }
865:
866: /*
867: * Sink the top element to its correct position
868: */
869: static void
870: file_header_heap_sink(struct file_header **fh, size_t indx, size_t size)
871: {
872: size_t left_child_index;
873: size_t right_child_index;
874:
875: left_child_index = indx + indx + 1;
876: right_child_index = left_child_index + 1;
877:
878: if (left_child_index < size) {
879: size_t min_child_index;
880:
881: min_child_index = left_child_index;
882:
883: if ((right_child_index < size) &&
884: (file_header_cmp(fh[left_child_index],
885: fh[right_child_index]) > 0))
886: min_child_index = right_child_index;
887: if (file_header_cmp(fh[indx], fh[min_child_index]) > 0) {
888: file_header_swap(fh, indx, min_child_index);
889: file_header_heap_sink(fh, min_child_index, size);
890: }
891: }
892: }
893:
894: /* <<== heap algorithm */
895:
896: /*
897: * Adds element to the "left" end
898: */
899: static void
900: file_header_list_rearrange_from_header(struct file_header **fh, size_t size)
901: {
902: file_header_heap_sink(fh, 0, size);
903: }
904:
905: /*
906: * Adds element to the "right" end
907: */
908: static void
909: file_header_list_push(struct file_header *f, struct file_header **fh, size_t size)
910: {
911: fh[size++] = f;
912: file_header_heap_swim(fh, size - 1);
913: }
914:
915: struct last_printed
916: {
917: struct bwstring *str;
918: };
919:
920: /*
921: * Prints the current line of the file
922: */
923: static void
924: file_header_print(struct file_header *fh, FILE *f_out, struct last_printed *lp)
925: {
1.12 millert 926: if (sort_opts_vals.uflag) {
927: if ((lp->str == NULL) || (str_list_coll(lp->str, &(fh->si)))) {
1.1 millert 928: bwsfwrite(fh->si->str, f_out, sort_opts_vals.zflag);
1.12 millert 929: if (lp->str)
930: bwsfree(lp->str);
931: lp->str = bwsdup(fh->si->str);
932: }
933: } else
934: bwsfwrite(fh->si->str, f_out, sort_opts_vals.zflag);
1.1 millert 935: }
936:
937: /*
938: * Read next line
939: */
940: static void
941: file_header_read_next(struct file_header *fh)
942: {
1.12 millert 943: struct bwstring *tmp;
1.1 millert 944:
1.12 millert 945: tmp = file_reader_readline(fh->fr);
946: if (tmp == NULL) {
947: file_reader_free(fh->fr);
948: fh->fr = NULL;
949: if (fh->si) {
950: sort_list_item_clean(fh->si);
951: sort_free(fh->si);
952: fh->si = NULL;
1.1 millert 953: }
1.12 millert 954: } else {
955: if (fh->si == NULL)
956: fh->si = sort_list_item_alloc();
957: sort_list_item_set(fh->si, tmp);
1.1 millert 958: }
959: }
960:
961: /*
962: * Merge array of "files headers"
963: */
964: static void
965: file_headers_merge(size_t fnum, struct file_header **fh, FILE *f_out)
966: {
967: struct last_printed lp;
968: size_t i;
969:
970: memset(&lp, 0, sizeof(lp));
971:
972: /*
1.19 ! deraadt 973: * construct the initial sort structure
1.1 millert 974: */
975: for (i = 0; i < fnum; i++)
976: file_header_list_push(fh[i], fh, i);
977:
978: while (fh[0]->fr) { /* unfinished files are always in front */
979: /* output the smallest line: */
980: file_header_print(fh[0], f_out, &lp);
981: /* read a new line, if possible: */
982: file_header_read_next(fh[0]);
983: /* re-arrange the list: */
984: file_header_list_rearrange_from_header(fh, fnum);
985: }
986:
987: if (lp.str)
988: bwsfree(lp.str);
989: }
990:
991: /*
992: * Merges the given files into the output file, which can be
993: * stdout.
994: */
995: static void
996: merge_files_array(size_t argc, char **argv, const char *fn_out)
997: {
1.12 millert 998: struct file_header **fh;
999: FILE *f_out;
1000: size_t i;
1.1 millert 1001:
1.12 millert 1002: f_out = openfile(fn_out, "w");
1.1 millert 1003:
1.12 millert 1004: if (f_out == NULL)
1005: err(2, "%s", fn_out);
1.1 millert 1006:
1.12 millert 1007: fh = sort_reallocarray(NULL, argc + 1, sizeof(struct file_header *));
1.1 millert 1008:
1.12 millert 1009: for (i = 0; i < argc; i++)
1010: file_header_init(fh + i, argv[i], i);
1.1 millert 1011:
1.12 millert 1012: file_headers_merge(argc, fh, f_out);
1.1 millert 1013:
1.12 millert 1014: for (i = 0; i < argc; i++)
1015: file_header_close(fh + i);
1.1 millert 1016:
1.12 millert 1017: sort_free(fh);
1.1 millert 1018:
1.12 millert 1019: closefile(f_out, fn_out);
1.1 millert 1020: }
1021:
1022: /*
1023: * Shrinks the file list until its size smaller than max number of opened files
1024: */
1025: static int
1026: shrink_file_list(struct file_list *fl)
1027: {
1.12 millert 1028: struct file_list new_fl;
1029: size_t indx = 0;
1030:
1031: if (fl->count < max_open_files)
1.1 millert 1032: return 0;
1033:
1.12 millert 1034: file_list_init(&new_fl, true);
1035: while (indx < fl->count) {
1036: char *fnew;
1037: size_t num;
1038:
1039: num = fl->count - indx;
1040: fnew = new_tmp_file_name();
1041:
1042: if (num >= max_open_files)
1043: num = max_open_files - 1;
1044: merge_files_array(num, fl->fns + indx, fnew);
1045: if (fl->tmp) {
1046: size_t i;
1047:
1048: for (i = 0; i < num; i++)
1049: unlink(fl->fns[indx + i]);
1.1 millert 1050: }
1.12 millert 1051: file_list_add(&new_fl, fnew, false);
1052: indx += num;
1053: }
1054: fl->tmp = false; /* already taken care of */
1055: file_list_clean(fl);
1.1 millert 1056:
1.12 millert 1057: fl->count = new_fl.count;
1058: fl->fns = new_fl.fns;
1059: fl->sz = new_fl.sz;
1060: fl->tmp = new_fl.tmp;
1.1 millert 1061:
1.12 millert 1062: return 1;
1.1 millert 1063: }
1064:
1065: /*
1066: * Merge list of files
1067: */
1068: void
1069: merge_files(struct file_list *fl, const char *fn_out)
1070: {
1.12 millert 1071: while (shrink_file_list(fl))
1072: ;
1.1 millert 1073:
1.12 millert 1074: merge_files_array(fl->count, fl->fns, fn_out);
1.1 millert 1075: }
1076:
1077: static const char *
1078: get_sort_method_name(int sm)
1079: {
1080: if (sm == SORT_MERGESORT)
1081: return "mergesort";
1082: else if (sort_opts_vals.sort_method == SORT_RADIXSORT)
1083: return "radixsort";
1084: else if (sort_opts_vals.sort_method == SORT_HEAPSORT)
1085: return "heapsort";
1086: else
1087: return "quicksort";
1088: }
1089:
1090: /*
1091: * Sort list of lines and writes it to the file
1092: */
1093: void
1094: sort_list_to_file(struct sort_list *list, const char *outfile)
1095: {
1096: struct sort_mods *sm = &(keys[0].sm);
1097:
1098: if (!sm->Mflag && !sm->Rflag && !sm->Vflag &&
1099: !sm->gflag && !sm->hflag && !sm->nflag) {
1100: if ((sort_opts_vals.sort_method == SORT_DEFAULT) && byte_sort)
1101: sort_opts_vals.sort_method = SORT_RADIXSORT;
1102:
1103: } else if (sort_opts_vals.sort_method == SORT_RADIXSORT)
1104: err(2, "Radix sort cannot be used with these sort options");
1105:
1106: /*
1.12 millert 1107: * To handle stable sort and the unique cases in the
1108: * right order, we need to use a stable algorithm.
1.1 millert 1109: */
1110: if (sort_opts_vals.sflag) {
1111: switch (sort_opts_vals.sort_method){
1112: case SORT_MERGESORT:
1113: break;
1114: case SORT_RADIXSORT:
1115: break;
1116: case SORT_DEFAULT:
1117: sort_opts_vals.sort_method = SORT_MERGESORT;
1118: break;
1119: default:
1120: errx(2, "The chosen sort method cannot be used with "
1121: "stable and/or unique sort");
1122: };
1123: }
1124:
1125: if (sort_opts_vals.sort_method == SORT_DEFAULT)
1126: sort_opts_vals.sort_method = DEFAULT_SORT_ALGORITHM;
1127:
1128: if (debug_sort)
1129: printf("sort_method=%s\n",
1130: get_sort_method_name(sort_opts_vals.sort_method));
1131:
1132: switch (sort_opts_vals.sort_method){
1133: case SORT_RADIXSORT:
1134: rxsort(list->list, list->count);
1135: break;
1136: case SORT_MERGESORT:
1137: mergesort(list->list, list->count,
1138: sizeof(struct sort_list_item *), list_coll);
1139: break;
1140: case SORT_HEAPSORT:
1141: heapsort(list->list, list->count,
1142: sizeof(struct sort_list_item *), list_coll);
1143: break;
1144: case SORT_QSORT:
1145: qsort(list->list, list->count,
1146: sizeof(struct sort_list_item *), list_coll);
1147: break;
1148: default:
1149: DEFAULT_SORT_FUNC(list->list, list->count,
1150: sizeof(struct sort_list_item *), list_coll);
1151: break;
1152: }
1153: sort_list_dump(list, outfile);
1154: }