![[BACK]](/icons/back.gif){kind=icon}
Return to file.c CVS log ![[TXT]](/icons/text.gif){kind=icon}
![[DIR]](/icons/dir.gif){kind=icon}
Up to [local] / src / usr.bin / sort|
Return to file.c CVS log | Up to [local] / src / usr.bin / sort |
1.4 ! millert 1: /* $OpenBSD: file.c,v 1.3 2015/03/20 00:26:38 millert Exp $ */
1.1 millert 2:
3: /*-
4: * Copyright (C) 2009 Gabor Kovesdan <gabor@FreeBSD.org>
5: * Copyright (C) 2012 Oleg Moskalenko <mom040267@gmail.com>
6: * All rights reserved.
7: *
8: * Redistribution and use in source and binary forms, with or without
9: * modification, are permitted provided that the following conditions
10: * are met:
11: * 1. Redistributions of source code must retain the above copyright
12: * notice, this list of conditions and the following disclaimer.
13: * 2. Redistributions in binary form must reproduce the above copyright
14: * notice, this list of conditions and the following disclaimer in the
15: * documentation and/or other materials provided with the distribution.
16: *
17: * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18: * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19: * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20: * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21: * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22: * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23: * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24: * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25: * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26: * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27: * SUCH DAMAGE.
28: */
29:
30: #include <sys/mman.h>
31: #include <sys/stat.h>
32: #include <sys/types.h>
33: #include <sys/queue.h>
34:
35: #include <err.h>
36: #include <fcntl.h>
37: #include <stdio.h>
38: #include <stdlib.h>
39: #include <string.h>
40: #include <unistd.h>
41: #include <wchar.h>
42: #include <wctype.h>
43:
44: #include "coll.h"
45: #include "file.h"
46: #include "radixsort.h"
47:
48: unsigned long long free_memory = 1000000;
49: unsigned long long available_free_memory = 1000000;
50:
51: bool use_mmap;
52:
53: const char *tmpdir = "/var/tmp";
54: const char *compress_program;
55:
56: size_t max_open_files = 16;
57:
58: /*
59: * How much space we read from file at once
60: */
61: #define READ_CHUNK 4096
62:
63: /*
64: * File reader structure
65: */
66: struct file_reader {
67: struct reader_buffer rb;
68: FILE *file;
69: char *fname;
70: unsigned char *buffer;
71: unsigned char *mmapaddr;
72: unsigned char *mmapptr;
73: size_t bsz;
74: size_t cbsz;
75: size_t mmapsize;
76: size_t strbeg;
77: int fd;
78: char elsymb;
79: };
80:
81: /*
82: * Structure to be used in file merge process.
83: */
84: struct file_header {
85: struct file_reader *fr;
86: struct sort_list_item *si; /* current top line */
87: size_t file_pos;
88: };
89:
90: /*
91: * List elements of "cleanable" files list.
92: */
93: struct CLEANABLE_FILE {
94: char *fn;
95: LIST_ENTRY(CLEANABLE_FILE) files;
96: };
97:
98: /*
99: * List header of "cleanable" files list.
100: */
101: static LIST_HEAD(CLEANABLE_FILES, CLEANABLE_FILE) tmp_files;
102:
103: /*
104: * Init tmp files list
105: */
106: void
107: init_tmp_files(void)
108: {
109:
110: LIST_INIT(&tmp_files);
111: }
112:
113: /*
114: * Save name of a tmp file for signal cleanup
115: */
116: void
117: tmp_file_atexit(const char *tmp_file)
118: {
119:
120: if (tmp_file) {
121: struct CLEANABLE_FILE *item =
122: sort_malloc(sizeof(struct CLEANABLE_FILE));
123: item->fn = sort_strdup(tmp_file);
124: LIST_INSERT_HEAD(&tmp_files, item, files);
125: }
126: }
127:
128: /*
129: * Clear tmp files
130: */
131: void
132: clear_tmp_files(void)
133: {
134: struct CLEANABLE_FILE *item;
135:
136: LIST_FOREACH(item, &tmp_files, files) {
137: if (item != NULL && item->fn != NULL)
138: unlink(item->fn);
139: }
140: }
141:
142: /*
143: * Check whether a file is a temporary file
144: */
145: static bool
146: file_is_tmp(const char *fn)
147: {
148: struct CLEANABLE_FILE *item;
149: bool ret = false;
150:
151: if (fn) {
152: LIST_FOREACH(item, &tmp_files, files) {
153: if (item != NULL && item->fn != NULL)
154: if (strcmp(item->fn, fn) == 0) {
155: ret = true;
156: break;
157: }
158: }
159: }
160:
161: return ret;
162: }
163:
164: /*
165: * Generate new temporary file name
166: */
167: char *
168: new_tmp_file_name(void)
169: {
170: static size_t tfcounter = 0;
171: static const char *fn = ".bsdsort.";
172: char *ret;
173: int len;
174:
1.4 ! millert 175: len = sort_asprintf(&ret, "%s/%s%d.%lu", tmpdir, fn, (int)getpid(),
1.1 millert 176: (unsigned long)(tfcounter++));
177: tmp_file_atexit(ret);
178: return ret;
179: }
180:
181: /*
182: * Initialize file list
183: */
184: void
185: file_list_init(struct file_list *fl, bool tmp)
186: {
187:
188: if (fl) {
189: fl->count = 0;
190: fl->sz = 0;
191: fl->fns = NULL;
192: fl->tmp = tmp;
193: }
194: }
195:
196: /*
197: * Add a file name to the list
198: */
199: void
200: file_list_add(struct file_list *fl, char *fn, bool allocate)
201: {
202:
203: if (fl && fn) {
204: if (fl->count >= fl->sz || fl->fns == NULL) {
205: fl->sz = fl->sz * 2 + 1;
206: fl->fns = sort_reallocarray(fl->fns, fl->sz,
207: sizeof(char *));
208: }
209: fl->fns[fl->count] = allocate ? sort_strdup(fn) : fn;
210: fl->count += 1;
211: }
212: }
213:
214: /*
215: * Populate file list from array of file names
216: */
217: void
218: file_list_populate(struct file_list *fl, int argc, char **argv, bool allocate)
219: {
220:
221: if (fl && argv) {
222: int i;
223:
224: for (i = 0; i < argc; i++)
225: file_list_add(fl, argv[i], allocate);
226: }
227: }
228:
229: /*
230: * Clean file list data and delete the files,
231: * if this is a list of temporary files
232: */
233: void
234: file_list_clean(struct file_list *fl)
235: {
236:
237: if (fl) {
238: if (fl->fns) {
239: size_t i;
240:
241: for (i = 0; i < fl->count; i++) {
242: if (fl->fns[i]) {
243: if (fl->tmp)
244: unlink(fl->fns[i]);
245: sort_free(fl->fns[i]);
246: fl->fns[i] = 0;
247: }
248: }
249: sort_free(fl->fns);
250: fl->fns = NULL;
251: }
252: fl->sz = 0;
253: fl->count = 0;
254: fl->tmp = false;
255: }
256: }
257:
258: /*
259: * Init sort list
260: */
261: void
262: sort_list_init(struct sort_list *l)
263: {
264:
265: if (l) {
266: l->count = 0;
267: l->size = 0;
268: l->memsize = sizeof(struct sort_list);
269: l->list = NULL;
270: }
271: }
272:
273: /*
274: * Add string to sort list
275: */
276: void
277: sort_list_add(struct sort_list *l, struct bwstring *str)
278: {
279:
280: if (l && str) {
281: size_t indx = l->count;
282:
283: if ((l->list == NULL) || (indx >= l->size)) {
284: size_t newsize = (l->size + 1) + 1024;
285:
286: l->list = sort_reallocarray(l->list, newsize,
287: sizeof(struct sort_list_item *));
288: l->memsize += (newsize - l->size) *
289: sizeof(struct sort_list_item *);
290: l->size = newsize;
291: }
292: l->list[indx] = sort_list_item_alloc();
293: sort_list_item_set(l->list[indx], str);
294: l->memsize += sort_list_item_size(l->list[indx]);
295: l->count += 1;
296: }
297: }
298:
299: /*
300: * Clean sort list data
301: */
302: void
303: sort_list_clean(struct sort_list *l)
304: {
305:
306: if (l) {
307: if (l->list) {
308: size_t i;
309:
310: for (i = 0; i < l->count; i++) {
311: struct sort_list_item *item;
312:
313: item = l->list[i];
314:
315: if (item) {
316: sort_list_item_clean(item);
317: sort_free(item);
318: l->list[i] = NULL;
319: }
320: }
321: sort_free(l->list);
322: l->list = NULL;
323: }
324: l->count = 0;
325: l->size = 0;
326: l->memsize = sizeof(struct sort_list);
327: }
328: }
329:
330: /*
331: * Write sort list to file
332: */
333: void
334: sort_list_dump(struct sort_list *l, const char *fn)
335: {
336:
337: if (l && fn) {
338: FILE *f;
339:
340: f = openfile(fn, "w");
341: if (f == NULL)
342: err(2, "%s", fn);
343:
344: if (l->list) {
345: size_t i;
346: if (!sort_opts_vals.uflag) {
347: for (i = 0; i < l->count; ++i)
348: bwsfwrite(l->list[i]->str, f,
349: sort_opts_vals.zflag);
350: } else {
351: struct sort_list_item *last_printed_item = NULL;
352: struct sort_list_item *item;
353: for (i = 0; i < l->count; ++i) {
354: item = l->list[i];
355: if ((last_printed_item == NULL) ||
356: list_coll(&last_printed_item, &item)) {
357: bwsfwrite(item->str, f, sort_opts_vals.zflag);
358: last_printed_item = item;
359: }
360: }
361: }
362: }
363:
364: closefile(f, fn);
365: }
366: }
367:
368: /*
369: * Checks if the given file is sorted. Stops at the first disorder,
370: * prints the disordered line and returns 1.
371: */
372: int
373: check(const char *fn)
374: {
375: struct bwstring *s1, *s2, *s1disorder, *s2disorder;
376: struct file_reader *fr;
377: struct keys_array *ka1, *ka2;
378: int res;
379: size_t pos, posdisorder;
380:
381: s1 = s2 = s1disorder = s2disorder = NULL;
382: ka1 = ka2 = NULL;
383:
384: fr = file_reader_init(fn);
385:
386: res = 0;
387: pos = 1;
388: posdisorder = 1;
389:
390: if (fr == NULL) {
391: err(2, "%s", fn);
392: goto end;
393: }
394:
395: s1 = file_reader_readline(fr);
396: if (s1 == NULL)
397: goto end;
398:
399: ka1 = keys_array_alloc();
400: preproc(s1, ka1);
401:
402: s2 = file_reader_readline(fr);
403: if (s2 == NULL)
404: goto end;
405:
406: ka2 = keys_array_alloc();
407: preproc(s2, ka2);
408:
409: for (;;) {
410:
411: if (debug_sort) {
412: bwsprintf(stdout, s2, "s1=<", ">");
413: bwsprintf(stdout, s1, "s2=<", ">");
414: }
415: int cmp = key_coll(ka2, ka1, 0);
416: if (debug_sort)
417: printf("; cmp1=%d", cmp);
418:
419: if (!cmp && sort_opts_vals.complex_sort &&
420: !(sort_opts_vals.uflag) && !(sort_opts_vals.sflag)) {
421: cmp = top_level_str_coll(s2, s1);
422: if (debug_sort)
423: printf("; cmp2=%d", cmp);
424: }
425: if (debug_sort)
426: printf("\n");
427:
428: if ((sort_opts_vals.uflag && (cmp <= 0)) || (cmp < 0)) {
429: if (!(sort_opts_vals.csilentflag)) {
430: s2disorder = bwsdup(s2);
431: posdisorder = pos;
432: if (debug_sort)
433: s1disorder = bwsdup(s1);
434: }
435: res = 1;
436: goto end;
437: }
438:
439: pos++;
440:
441: clean_keys_array(s1, ka1);
442: sort_free(ka1);
443: ka1 = ka2;
444: ka2 = NULL;
445:
446: bwsfree(s1);
447: s1 = s2;
448:
449: s2 = file_reader_readline(fr);
450: if (s2 == NULL)
451: goto end;
452:
453: ka2 = keys_array_alloc();
454: preproc(s2, ka2);
455: }
456:
457: end:
458: if (ka1) {
459: clean_keys_array(s1, ka1);
460: sort_free(ka1);
461: }
462:
463: if (s1)
464: bwsfree(s1);
465:
466: if (ka2) {
467: clean_keys_array(s2, ka2);
468: sort_free(ka2);
469: }
470:
471: if (s2)
472: bwsfree(s2);
473:
474: if (fn == NULL || *fn == 0 || strcmp(fn, "-") == 0) {
475: for (;;) {
476: s2 = file_reader_readline(fr);
477: if (s2 == NULL)
478: break;
479: bwsfree(s2);
480: }
481: }
482:
483: file_reader_free(fr);
484:
485: if (s2disorder) {
486: bws_disorder_warnx(s2disorder, fn, posdisorder);
487: if (s1disorder) {
488: bws_disorder_warnx(s1disorder, fn, posdisorder);
489: if (s1disorder != s2disorder)
490: bwsfree(s1disorder);
491: }
492: bwsfree(s2disorder);
493: s1disorder = NULL;
494: s2disorder = NULL;
495: }
496:
497: if (res)
498: exit(res);
499:
500: return 0;
501: }
502:
503: /*
504: * Opens a file. If the given filename is "-", stdout will be
505: * opened.
506: */
507: FILE *
508: openfile(const char *fn, const char *mode)
509: {
510: FILE *file;
511:
512: if (strcmp(fn, "-") == 0) {
513: return (mode && mode[0] == 'r') ? stdin : stdout;
514: } else {
515: mode_t orig_file_mask = 0;
516: int is_tmp = file_is_tmp(fn);
517:
518: if (is_tmp && (mode[0] == 'w'))
519: orig_file_mask = umask(S_IWGRP | S_IWOTH |
520: S_IRGRP | S_IROTH);
521:
522: if (is_tmp && (compress_program != NULL)) {
523: char *cmd;
524: int len;
525:
526: fflush(stdout);
527:
528: if (mode[0] == 'r')
1.4 ! millert 529: len = sort_asprintf(&cmd, "cat %s | %s -d",
1.1 millert 530: fn, compress_program);
531: else if (mode[0] == 'w')
1.4 ! millert 532: len = sort_asprintf(&cmd, "%s > %s",
1.1 millert 533: compress_program, fn);
534: else
535: err(2, "Wrong file mode");
536:
537: if ((file = popen(cmd, mode)) == NULL)
538: err(2, NULL);
539:
540: sort_free(cmd);
541:
542: } else if ((file = fopen(fn, mode)) == NULL)
543: err(2, "%s", fn);
544:
545: if (is_tmp && (mode[0] == 'w'))
546: umask(orig_file_mask);
547: }
548:
549: return file;
550: }
551:
552: /*
553: * Close file
554: */
555: void
556: closefile(FILE *f, const char *fn)
557: {
558: if (f == NULL) {
559: ;
560: } else if (f == stdin) {
561: ;
562: } else if (f == stdout) {
563: fflush(f);
564: } else {
565: if (file_is_tmp(fn) && compress_program != NULL) {
566: if (pclose(f) < 0)
567: err(2, NULL);
568: } else
569: fclose(f);
570: }
571: }
572:
573: /*
574: * Reads a file into the internal buffer.
575: */
576: struct file_reader *
577: file_reader_init(const char *fsrc)
578: {
579: struct file_reader *ret;
580:
581: if (fsrc == NULL)
582: fsrc = "-";
583:
1.2 millert 584: ret = sort_calloc(1, sizeof(struct file_reader));
1.1 millert 585:
586: ret->elsymb = '\n';
587: if (sort_opts_vals.zflag)
588: ret->elsymb = 0;
589:
590: ret->fname = sort_strdup(fsrc);
591:
592: if (strcmp(fsrc, "-") && (compress_program == NULL) && use_mmap) {
593: struct stat stat_buf;
594: void *addr;
595: size_t sz = 0;
596: int fd;
597:
598: fd = open(fsrc, O_RDONLY);
599: if (fd < 0)
600: err(2, "%s", fsrc);
601:
602: if (fstat(fd, &stat_buf) < 0)
603: err(2, "%s", fsrc);
604: sz = stat_buf.st_size;
605:
606: addr = mmap(NULL, sz, PROT_READ, 0, fd, 0);
607: if (addr == MAP_FAILED) {
608: close(fd);
609: } else {
610: ret->fd = fd;
611: ret->mmapaddr = addr;
612: ret->mmapsize = sz;
613: ret->mmapptr = ret->mmapaddr;
614: posix_madvise(addr, sz, POSIX_MADV_SEQUENTIAL);
615: }
616: }
617:
618: if (ret->mmapaddr == NULL) {
619: ret->file = openfile(fsrc, "r");
620: if (ret->file == NULL)
621: err(2, "%s", fsrc);
622:
623: if (strcmp(fsrc, "-")) {
624: ret->cbsz = READ_CHUNK;
625: ret->buffer = sort_malloc(ret->cbsz);
626: ret->bsz = 0;
627: ret->strbeg = 0;
628:
629: ret->bsz = fread(ret->buffer, 1, ret->cbsz, ret->file);
630: if (ret->bsz == 0) {
631: if (ferror(ret->file))
632: err(2, NULL);
633: }
634: }
635: }
636:
637: return ret;
638: }
639:
640: struct bwstring *
641: file_reader_readline(struct file_reader *fr)
642: {
643: struct bwstring *ret = NULL;
644:
645: if (fr->mmapaddr) {
646: unsigned char *mmapend;
647:
648: mmapend = fr->mmapaddr + fr->mmapsize;
649: if (fr->mmapptr >= mmapend)
650: return NULL;
651: else {
652: unsigned char *strend;
653: size_t sz;
654:
655: sz = mmapend - fr->mmapptr;
656: strend = memchr(fr->mmapptr, fr->elsymb, sz);
657:
658: if (strend == NULL) {
659: ret = bwscsbdup(fr->mmapptr, sz);
660: fr->mmapptr = mmapend;
661: } else {
662: ret = bwscsbdup(fr->mmapptr, strend -
663: fr->mmapptr);
664: fr->mmapptr = strend + 1;
665: }
666: }
667:
668: } else if (fr->file != stdin) {
669: unsigned char *strend;
670: size_t bsz1, remsz, search_start;
671:
672: search_start = 0;
673: remsz = 0;
674: strend = NULL;
675:
676: if (fr->bsz > fr->strbeg)
677: remsz = fr->bsz - fr->strbeg;
678:
679: /* line read cycle */
680: for (;;) {
681: if (remsz > search_start)
682: strend = memchr(fr->buffer + fr->strbeg +
683: search_start, fr->elsymb, remsz -
684: search_start);
685: else
686: strend = NULL;
687:
688: if (strend)
689: break;
690: if (feof(fr->file))
691: break;
692:
693: if (fr->bsz != fr->cbsz)
694: /* NOTREACHED */
695: err(2, "File read software error 1");
696:
697: if (remsz > (READ_CHUNK >> 1)) {
698: search_start = fr->cbsz - fr->strbeg;
699: fr->cbsz += READ_CHUNK;
700: fr->buffer = sort_realloc(fr->buffer,
701: fr->cbsz);
702: bsz1 = fread(fr->buffer + fr->bsz, 1,
703: READ_CHUNK, fr->file);
704: if (bsz1 == 0) {
705: if (ferror(fr->file))
706: err(2, NULL);
707: break;
708: }
709: fr->bsz += bsz1;
710: remsz += bsz1;
711: } else {
712: if (remsz > 0 && fr->strbeg>0)
713: bcopy(fr->buffer + fr->strbeg,
714: fr->buffer, remsz);
715:
716: fr->strbeg = 0;
717: search_start = remsz;
718: bsz1 = fread(fr->buffer + remsz, 1,
719: fr->cbsz - remsz, fr->file);
720: if (bsz1 == 0) {
721: if (ferror(fr->file))
722: err(2, NULL);
723: break;
724: }
725: fr->bsz = remsz + bsz1;
726: remsz = fr->bsz;
727: }
728: }
729:
730: if (strend == NULL)
731: strend = fr->buffer + fr->bsz;
732:
733: if ((fr->buffer + fr->strbeg <= strend) &&
734: (fr->strbeg < fr->bsz) && (remsz>0))
735: ret = bwscsbdup(fr->buffer + fr->strbeg, strend -
736: fr->buffer - fr->strbeg);
737:
738: fr->strbeg = (strend - fr->buffer) + 1;
739:
740: } else {
741: size_t len = 0;
742:
743: ret = bwsfgetln(fr->file, &len, sort_opts_vals.zflag,
744: &(fr->rb));
745: }
746:
747: return ret;
748: }
749:
750: static void
751: file_reader_clean(struct file_reader *fr)
752: {
753:
754: if (fr) {
755: if (fr->mmapaddr)
756: munmap(fr->mmapaddr, fr->mmapsize);
757:
758: if (fr->fd)
759: close(fr->fd);
760:
761: if (fr->buffer)
762: sort_free(fr->buffer);
763:
764: if (fr->file)
765: if (fr->file != stdin)
766: closefile(fr->file, fr->fname);
767:
768: if (fr->fname)
769: sort_free(fr->fname);
770:
771: memset(fr, 0, sizeof(struct file_reader));
772: }
773: }
774:
775: void
776: file_reader_free(struct file_reader *fr)
777: {
778:
779: if (fr) {
780: file_reader_clean(fr);
781: sort_free(fr);
782: }
783: }
784:
785: int
786: procfile(const char *fsrc, struct sort_list *list, struct file_list *fl)
787: {
788: struct file_reader *fr;
789:
790: fr = file_reader_init(fsrc);
791: if (fr == NULL)
792: err(2, "%s", fsrc);
793:
794: /* file browse cycle */
795: for (;;) {
796: struct bwstring *bws;
797:
798: bws = file_reader_readline(fr);
799:
800: if (bws == NULL)
801: break;
802:
803: sort_list_add(list, bws);
804:
805: if (list->memsize >= available_free_memory) {
806: char *fn;
807:
808: fn = new_tmp_file_name();
809: sort_list_to_file(list, fn);
810: file_list_add(fl, fn, false);
811: sort_list_clean(list);
812: }
813: }
814:
815: file_reader_free(fr);
816:
817: return 0;
818: }
819:
820: /*
821: * Compare file headers. Files with EOF always go to the end of the list.
822: */
823: static int
824: file_header_cmp(struct file_header *f1, struct file_header *f2)
825: {
826:
827: if (f1 == f2)
828: return 0;
829: else {
830: if (f1->fr == NULL) {
831: return (f2->fr == NULL) ? 0 : 1;
832: } else if (f2->fr == NULL)
833: return -1;
834: else {
835: int ret;
836:
837: ret = list_coll(&(f1->si), &(f2->si));
838: if (!ret)
839: return (f1->file_pos < f2->file_pos) ? -1 : 1;
840: return ret;
841: }
842: }
843: }
844:
845: /*
846: * Allocate and init file header structure
847: */
848: static void
849: file_header_init(struct file_header **fh, const char *fn, size_t file_pos)
850: {
851:
852: if (fh && fn) {
853: struct bwstring *line;
854:
855: *fh = sort_malloc(sizeof(struct file_header));
856: (*fh)->file_pos = file_pos;
857: (*fh)->fr = file_reader_init(fn);
858: if ((*fh)->fr == NULL) {
859: err(2, "Cannot open %s for reading",
860: strcmp(fn, "-") == 0 ? "stdin" : fn);
861: }
862: line = file_reader_readline((*fh)->fr);
863: if (line == NULL) {
864: file_reader_free((*fh)->fr);
865: (*fh)->fr = NULL;
866: (*fh)->si = NULL;
867: } else {
868: (*fh)->si = sort_list_item_alloc();
869: sort_list_item_set((*fh)->si, line);
870: }
871: }
872: }
873:
874: /*
875: * Close file
876: */
877: static void
878: file_header_close(struct file_header **fh)
879: {
880:
881: if (fh && *fh) {
882: if ((*fh)->fr) {
883: file_reader_free((*fh)->fr);
884: (*fh)->fr = NULL;
885: }
886: if ((*fh)->si) {
887: sort_list_item_clean((*fh)->si);
888: sort_free((*fh)->si);
889: (*fh)->si = NULL;
890: }
891: sort_free(*fh);
892: *fh = NULL;
893: }
894: }
895:
896: /*
897: * Swap two array elements
898: */
899: static void
900: file_header_swap(struct file_header **fh, size_t i1, size_t i2)
901: {
902: struct file_header *tmp;
903:
904: tmp = fh[i1];
905: fh[i1] = fh[i2];
906: fh[i2] = tmp;
907: }
908:
909: /* heap algorithm ==>> */
910:
911: /*
912: * See heap sort algorithm
913: * "Raises" last element to its right place
914: */
915: static void
916: file_header_heap_swim(struct file_header **fh, size_t indx)
917: {
918:
919: if (indx > 0) {
920: size_t parent_index;
921:
922: parent_index = (indx - 1) >> 1;
923:
924: if (file_header_cmp(fh[indx], fh[parent_index]) < 0) {
925: /* swap child and parent and continue */
926: file_header_swap(fh, indx, parent_index);
927: file_header_heap_swim(fh, parent_index);
928: }
929: }
930: }
931:
932: /*
933: * Sink the top element to its correct position
934: */
935: static void
936: file_header_heap_sink(struct file_header **fh, size_t indx, size_t size)
937: {
938: size_t left_child_index;
939: size_t right_child_index;
940:
941: left_child_index = indx + indx + 1;
942: right_child_index = left_child_index + 1;
943:
944: if (left_child_index < size) {
945: size_t min_child_index;
946:
947: min_child_index = left_child_index;
948:
949: if ((right_child_index < size) &&
950: (file_header_cmp(fh[left_child_index],
951: fh[right_child_index]) > 0))
952: min_child_index = right_child_index;
953: if (file_header_cmp(fh[indx], fh[min_child_index]) > 0) {
954: file_header_swap(fh, indx, min_child_index);
955: file_header_heap_sink(fh, min_child_index, size);
956: }
957: }
958: }
959:
960: /* <<== heap algorithm */
961:
962: /*
963: * Adds element to the "left" end
964: */
965: static void
966: file_header_list_rearrange_from_header(struct file_header **fh, size_t size)
967: {
968:
969: file_header_heap_sink(fh, 0, size);
970: }
971:
972: /*
973: * Adds element to the "right" end
974: */
975: static void
976: file_header_list_push(struct file_header *f, struct file_header **fh, size_t size)
977: {
978:
979: fh[size++] = f;
980: file_header_heap_swim(fh, size - 1);
981: }
982:
983: struct last_printed
984: {
985: struct bwstring *str;
986: };
987:
988: /*
989: * Prints the current line of the file
990: */
991: static void
992: file_header_print(struct file_header *fh, FILE *f_out, struct last_printed *lp)
993: {
994:
995: if (fh && fh->fr && f_out && fh->si && fh->si->str) {
996: if (sort_opts_vals.uflag) {
997: if ((lp->str == NULL) || (str_list_coll(lp->str, &(fh->si)))) {
998: bwsfwrite(fh->si->str, f_out, sort_opts_vals.zflag);
999: if (lp->str)
1000: bwsfree(lp->str);
1001: lp->str = bwsdup(fh->si->str);
1002: }
1003: } else
1004: bwsfwrite(fh->si->str, f_out, sort_opts_vals.zflag);
1005: }
1006: }
1007:
1008: /*
1009: * Read next line
1010: */
1011: static void
1012: file_header_read_next(struct file_header *fh)
1013: {
1014:
1015: if (fh && fh->fr) {
1016: struct bwstring *tmp;
1017:
1018: tmp = file_reader_readline(fh->fr);
1019: if (tmp == NULL) {
1020: file_reader_free(fh->fr);
1021: fh->fr = NULL;
1022: if (fh->si) {
1023: sort_list_item_clean(fh->si);
1024: sort_free(fh->si);
1025: fh->si = NULL;
1026: }
1027: } else {
1028: if (fh->si == NULL)
1029: fh->si = sort_list_item_alloc();
1030: sort_list_item_set(fh->si, tmp);
1031: }
1032: }
1033: }
1034:
1035: /*
1036: * Merge array of "files headers"
1037: */
1038: static void
1039: file_headers_merge(size_t fnum, struct file_header **fh, FILE *f_out)
1040: {
1041: struct last_printed lp;
1042: size_t i;
1043:
1044: memset(&lp, 0, sizeof(lp));
1045:
1046: /*
1047: * construct the initial sort structure
1048: */
1049: for (i = 0; i < fnum; i++)
1050: file_header_list_push(fh[i], fh, i);
1051:
1052: while (fh[0]->fr) { /* unfinished files are always in front */
1053: /* output the smallest line: */
1054: file_header_print(fh[0], f_out, &lp);
1055: /* read a new line, if possible: */
1056: file_header_read_next(fh[0]);
1057: /* re-arrange the list: */
1058: file_header_list_rearrange_from_header(fh, fnum);
1059: }
1060:
1061: if (lp.str)
1062: bwsfree(lp.str);
1063: }
1064:
1065: /*
1066: * Merges the given files into the output file, which can be
1067: * stdout.
1068: */
1069: static void
1070: merge_files_array(size_t argc, char **argv, const char *fn_out)
1071: {
1072:
1073: if (argv && fn_out) {
1074: struct file_header **fh;
1075: FILE *f_out;
1076: size_t i;
1077:
1078: f_out = openfile(fn_out, "w");
1079:
1080: if (f_out == NULL)
1081: err(2, "%s", fn_out);
1082:
1083: fh = sort_malloc((argc + 1) * sizeof(struct file_header *));
1084:
1085: for (i = 0; i < argc; i++)
1086: file_header_init(fh + i, argv[i], (size_t) i);
1087:
1088: file_headers_merge(argc, fh, f_out);
1089:
1090: for (i = 0; i < argc; i++)
1091: file_header_close(fh + i);
1092:
1093: sort_free(fh);
1094:
1095: closefile(f_out, fn_out);
1096: }
1097: }
1098:
1099: /*
1100: * Shrinks the file list until its size smaller than max number of opened files
1101: */
1102: static int
1103: shrink_file_list(struct file_list *fl)
1104: {
1105:
1106: if (fl == NULL || (size_t)fl->count < max_open_files)
1107: return 0;
1108: else {
1109: struct file_list new_fl;
1110: size_t indx = 0;
1111:
1112: file_list_init(&new_fl, true);
1113: while (indx < fl->count) {
1114: char *fnew;
1115: size_t num;
1116:
1117: num = fl->count - indx;
1118: fnew = new_tmp_file_name();
1119:
1120: if ((size_t) num >= max_open_files)
1121: num = max_open_files - 1;
1122: merge_files_array(num, fl->fns + indx, fnew);
1123: if (fl->tmp) {
1124: size_t i;
1125:
1126: for (i = 0; i < num; i++)
1127: unlink(fl->fns[indx + i]);
1128: }
1129: file_list_add(&new_fl, fnew, false);
1130: indx += num;
1131: }
1132: fl->tmp = false; /* already taken care of */
1133: file_list_clean(fl);
1134:
1135: fl->count = new_fl.count;
1136: fl->fns = new_fl.fns;
1137: fl->sz = new_fl.sz;
1138: fl->tmp = new_fl.tmp;
1139:
1140: return 1;
1141: }
1142: }
1143:
1144: /*
1145: * Merge list of files
1146: */
1147: void
1148: merge_files(struct file_list *fl, const char *fn_out)
1149: {
1150:
1151: if (fl && fn_out) {
1152: while (shrink_file_list(fl));
1153:
1154: merge_files_array(fl->count, fl->fns, fn_out);
1155: }
1156: }
1157:
1158: static const char *
1159: get_sort_method_name(int sm)
1160: {
1161:
1162: if (sm == SORT_MERGESORT)
1163: return "mergesort";
1164: else if (sort_opts_vals.sort_method == SORT_RADIXSORT)
1165: return "radixsort";
1166: else if (sort_opts_vals.sort_method == SORT_HEAPSORT)
1167: return "heapsort";
1168: else
1169: return "quicksort";
1170: }
1171:
1172: /*
1173: * Sort list of lines and writes it to the file
1174: */
1175: void
1176: sort_list_to_file(struct sort_list *list, const char *outfile)
1177: {
1178: struct sort_mods *sm = &(keys[0].sm);
1179:
1180: if (!sm->Mflag && !sm->Rflag && !sm->Vflag &&
1181: !sm->gflag && !sm->hflag && !sm->nflag) {
1182: if ((sort_opts_vals.sort_method == SORT_DEFAULT) && byte_sort)
1183: sort_opts_vals.sort_method = SORT_RADIXSORT;
1184:
1185: } else if (sort_opts_vals.sort_method == SORT_RADIXSORT)
1186: err(2, "Radix sort cannot be used with these sort options");
1187:
1188: /*
1189: * to handle stable sort and the unique cases in the
1190: * right order, we need stable basic algorithm
1191: */
1192: if (sort_opts_vals.sflag) {
1193: switch (sort_opts_vals.sort_method){
1194: case SORT_MERGESORT:
1195: break;
1196: case SORT_RADIXSORT:
1197: break;
1198: case SORT_DEFAULT:
1199: sort_opts_vals.sort_method = SORT_MERGESORT;
1200: break;
1201: default:
1202: errx(2, "The chosen sort method cannot be used with "
1203: "stable and/or unique sort");
1204: };
1205: }
1206:
1207: if (sort_opts_vals.sort_method == SORT_DEFAULT)
1208: sort_opts_vals.sort_method = DEFAULT_SORT_ALGORITHM;
1209:
1210: if (debug_sort)
1211: printf("sort_method=%s\n",
1212: get_sort_method_name(sort_opts_vals.sort_method));
1213:
1214: switch (sort_opts_vals.sort_method){
1215: case SORT_RADIXSORT:
1216: rxsort(list->list, list->count);
1217: break;
1218: case SORT_MERGESORT:
1219: mergesort(list->list, list->count,
1220: sizeof(struct sort_list_item *), list_coll);
1221: break;
1222: case SORT_HEAPSORT:
1223: heapsort(list->list, list->count,
1224: sizeof(struct sort_list_item *), list_coll);
1225: break;
1226: case SORT_QSORT:
1227: qsort(list->list, list->count,
1228: sizeof(struct sort_list_item *), list_coll);
1229: break;
1230: default:
1231: DEFAULT_SORT_FUNC(list->list, list->count,
1232: sizeof(struct sort_list_item *), list_coll);
1233: break;
1234: }
1235: sort_list_dump(list, outfile);
1236: }