=================================================================== RCS file: /cvsrepo/anoncvs/cvs/src/usr.bin/rsync/uploader.c,v retrieving revision 1.24 retrieving revision 1.24.2.1 diff -c -r1.24 -r1.24.2.1 *** src/usr.bin/rsync/uploader.c 2021/03/22 11:20:04 1.24 --- src/usr.bin/rsync/uploader.c 2021/11/09 13:41:24 1.24.2.1 *************** *** 1,4 **** ! /* $Id: uploader.c,v 1.24 2021/03/22 11:20:04 claudio Exp $ */ /* * Copyright (c) 2019 Kristaps Dzonsons * Copyright (c) 2019 Florian Obser --- 1,4 ---- ! /* $Id: uploader.c,v 1.24.2.1 2021/11/09 13:41:24 benno Exp $ */ /* * Copyright (c) 2019 Kristaps Dzonsons * Copyright (c) 2019 Florian Obser *************** *** 19,24 **** --- 19,25 ---- #include #include + #include #include #include #include *************** *** 33,40 **** enum uploadst { UPLOAD_FIND_NEXT = 0, /* find next to upload to sender */ ! UPLOAD_WRITE_LOCAL, /* wait to write to sender */ ! UPLOAD_READ_LOCAL, /* wait to read from local file */ UPLOAD_FINISHED /* nothing more to do in phase */ }; --- 34,40 ---- enum uploadst { UPLOAD_FIND_NEXT = 0, /* find next to upload to sender */ ! UPLOAD_WRITE, /* wait to write to sender */ UPLOAD_FINISHED /* nothing more to do in phase */ }; *************** *** 80,86 **** * operator that we're a link. */ static void ! log_link(struct sess *sess, const struct flist *f) { if (!sess->opts->server) --- 80,86 ---- * operator that we're a link. */ static void ! log_symlink(struct sess *sess, const struct flist *f) { if (!sess->opts->server) *************** *** 167,173 **** * Return <0 on failure 0 on success. */ static int ! pre_link(struct upload *p, struct sess *sess) { struct stat st; const struct flist *f; --- 167,173 ---- * Return <0 on failure 0 on success. */ static int ! pre_symlink(struct upload *p, struct sess *sess) { struct stat st; const struct flist *f; *************** *** 180,187 **** if (!sess->opts->preserve_links) { WARNX("%s: ignoring symlink", f->path); return 0; ! } else if (sess->opts->dry_run) { ! log_link(sess, f); return 0; } --- 180,188 ---- if (!sess->opts->preserve_links) { WARNX("%s: ignoring symlink", f->path); return 0; ! } ! if (sess->opts->dry_run) { ! log_symlink(sess, f); return 0; } *************** *** 194,199 **** --- 195,205 ---- assert(p->rootfd != -1); rc = fstatat(p->rootfd, f->path, &st, AT_SYMLINK_NOFOLLOW); + + if (rc == -1 && errno != ENOENT) { + ERR("%s: fstatat", f->path); + return -1; + } if (rc != -1 && !S_ISLNK(st.st_mode)) { if (S_ISDIR(st.st_mode) && unlinkat(p->rootfd, f->path, AT_REMOVEDIR) == -1) { *************** *** 201,209 **** return -1; } rc = -1; - } else if (rc == -1 && errno != ENOENT) { - ERR("%s: fstatat", f->path); - return -1; } /* --- 207,212 ---- *************** *** 259,270 **** free(temp); } ! log_link(sess, f); return 0; } /* ! * See pre_link(), but for devices. * FIXME: this is very similar to the other pre_xxx() functions. * Return <0 on failure 0 on success. */ --- 262,273 ---- free(temp); } ! log_symlink(sess, f); return 0; } /* ! * See pre_symlink(), but for devices. * FIXME: this is very similar to the other pre_xxx() functions. * Return <0 on failure 0 on success. */ *************** *** 282,288 **** if (!sess->opts->devices || getuid() != 0) { WARNX("skipping non-regular file %s", f->path); return 0; ! } else if (sess->opts->dry_run) { log_file(sess, f); return 0; } --- 285,292 ---- if (!sess->opts->devices || getuid() != 0) { WARNX("skipping non-regular file %s", f->path); return 0; ! } ! if (sess->opts->dry_run) { log_file(sess, f); return 0; } *************** *** 296,301 **** --- 300,309 ---- assert(p->rootfd != -1); rc = fstatat(p->rootfd, f->path, &st, AT_SYMLINK_NOFOLLOW); + if (rc == -1 && errno != ENOENT) { + ERR("%s: fstatat", f->path); + return -1; + } if (rc != -1 && !(S_ISBLK(st.st_mode) || S_ISCHR(st.st_mode))) { if (S_ISDIR(st.st_mode) && unlinkat(p->rootfd, f->path, AT_REMOVEDIR) == -1) { *************** *** 303,311 **** return -1; } rc = -1; - } else if (rc == -1 && errno != ENOENT) { - ERR("%s: fstatat", f->path); - return -1; } /* Make sure existing device is of the correct type. */ --- 311,316 ---- *************** *** 351,357 **** } /* ! * See pre_link(), but for FIFOs. * FIXME: this is very similar to the other pre_xxx() functions. * Return <0 on failure 0 on success. */ --- 356,362 ---- } /* ! * See pre_symlink(), but for FIFOs. * FIXME: this is very similar to the other pre_xxx() functions. * Return <0 on failure 0 on success. */ *************** *** 369,375 **** if (!sess->opts->specials) { WARNX("skipping non-regular file %s", f->path); return 0; ! } else if (sess->opts->dry_run) { log_file(sess, f); return 0; } --- 374,381 ---- if (!sess->opts->specials) { WARNX("skipping non-regular file %s", f->path); return 0; ! } ! if (sess->opts->dry_run) { log_file(sess, f); return 0; } *************** *** 383,388 **** --- 389,398 ---- assert(p->rootfd != -1); rc = fstatat(p->rootfd, f->path, &st, AT_SYMLINK_NOFOLLOW); + if (rc == -1 && errno != ENOENT) { + ERR("%s: fstatat", f->path); + return -1; + } if (rc != -1 && !S_ISFIFO(st.st_mode)) { if (S_ISDIR(st.st_mode) && unlinkat(p->rootfd, f->path, AT_REMOVEDIR) == -1) { *************** *** 390,398 **** return -1; } rc = -1; - } else if (rc == -1 && errno != ENOENT) { - ERR("%s: fstatat", f->path); - return -1; } if (rc == -1) { --- 400,405 ---- *************** *** 426,432 **** } /* ! * See pre_link(), but for socket files. * FIXME: this is very similar to the other pre_xxx() functions. * Return <0 on failure 0 on success. */ --- 433,439 ---- } /* ! * See pre_symlink(), but for socket files. * FIXME: this is very similar to the other pre_xxx() functions. * Return <0 on failure 0 on success. */ *************** *** 444,450 **** if (!sess->opts->specials) { WARNX("skipping non-regular file %s", f->path); return 0; ! } else if (sess->opts->dry_run) { log_file(sess, f); return 0; } --- 451,458 ---- if (!sess->opts->specials) { WARNX("skipping non-regular file %s", f->path); return 0; ! } ! if (sess->opts->dry_run) { log_file(sess, f); return 0; } *************** *** 458,463 **** --- 466,475 ---- assert(p->rootfd != -1); rc = fstatat(p->rootfd, f->path, &st, AT_SYMLINK_NOFOLLOW); + if (rc == -1 && errno != ENOENT) { + ERR("%s: fstatat", f->path); + return -1; + } if (rc != -1 && !S_ISSOCK(st.st_mode)) { if (S_ISDIR(st.st_mode) && unlinkat(p->rootfd, f->path, AT_REMOVEDIR) == -1) { *************** *** 465,473 **** return -1; } rc = -1; - } else if (rc == -1 && errno != ENOENT) { - ERR("%s: fstatat", f->path); - return -1; } if (rc == -1) { --- 477,482 ---- *************** *** 518,524 **** if (!sess->opts->recursive) { WARNX("%s: ignoring directory", f->path); return 0; ! } else if (sess->opts->dry_run) { log_dir(sess, f); return 0; } --- 527,534 ---- if (!sess->opts->recursive) { WARNX("%s: ignoring directory", f->path); return 0; ! } ! if (sess->opts->dry_run) { log_dir(sess, f); return 0; } *************** *** 529,535 **** if (rc == -1 && errno != ENOENT) { ERR("%s: fstatat", f->path); return -1; ! } else if (rc != -1 && !S_ISDIR(st.st_mode)) { ERRX("%s: not a directory", f->path); return -1; } else if (rc != -1) { --- 539,546 ---- if (rc == -1 && errno != ENOENT) { ERR("%s: fstatat", f->path); return -1; ! } ! if (rc != -1 && !S_ISDIR(st.st_mode)) { ERRX("%s: not a directory", f->path); return -1; } else if (rc != -1) { *************** *** 579,591 **** if (!sess->opts->recursive) return 1; ! else if (sess->opts->dry_run) return 1; if (fstatat(u->rootfd, f->path, &st, AT_SYMLINK_NOFOLLOW) == -1) { ERR("%s: fstatat", f->path); return 0; ! } else if (!S_ISDIR(st.st_mode)) { WARNX("%s: not a directory", f->path); return 0; } --- 590,603 ---- if (!sess->opts->recursive) return 1; ! if (sess->opts->dry_run) return 1; if (fstatat(u->rootfd, f->path, &st, AT_SYMLINK_NOFOLLOW) == -1) { ERR("%s: fstatat", f->path); return 0; ! } ! if (!S_ISDIR(st.st_mode)) { WARNX("%s: not a directory", f->path); return 0; } *************** *** 630,644 **** } /* * Try to open the file at the current index. ! * If the file does not exist, returns with success. * Return <0 on failure, 0 on success w/nothing to be done, >0 on * success and the file needs attention. */ static int ! pre_file(const struct upload *p, int *filefd, struct sess *sess) { const struct flist *f; f = &p->fl[p->idx]; assert(S_ISREG(f->st.mode)); --- 642,696 ---- } /* + * Check if file exists in the specified root directory. + * Returns: + * -1 on error + * 0 if file is considered the same + * 1 if file exists and is possible match + * 2 if file exists but quick check failed + * 3 if file does not exist + * The stat pointer st is only valid for 0, 1, and 2 returns. + */ + static int + check_file(int rootfd, const struct flist *f, struct stat *st) + { + if (fstatat(rootfd, f->path, st, AT_SYMLINK_NOFOLLOW) == -1) { + if (errno == ENOENT) + return 3; + + ERR("%s: fstatat", f->path); + return -1; + } + + /* non-regular file needs attention */ + if (!S_ISREG(st->st_mode)) + return 2; + + /* quick check if file is the same */ + /* TODO: add support for --checksum, --size-only and --ignore-times */ + if (st->st_size == f->st.size) { + if (st->st_mtime == f->st.mtime) + return 0; + return 1; + } + + /* file needs attention */ + return 2; + } + + /* * Try to open the file at the current index. ! * If the file does not exist, returns with >0. * Return <0 on failure, 0 on success w/nothing to be done, >0 on * success and the file needs attention. */ static int ! pre_file(const struct upload *p, int *filefd, off_t *size, ! struct sess *sess) { const struct flist *f; + struct stat st; + int i, rc, match = -1; f = &p->fl[p->idx]; assert(S_ISREG(f->st.mode)); *************** *** 652,672 **** return 0; } /* * For non dry-run cases, we'll write the acknowledgement later ! * in the rsync_uploader() function because we need to wait for ! * the open() call to complete. ! * If the call to openat() fails with ENOENT, there's a ! * fast-path between here and the write function, so we won't do ! * any blocking between now and then. */ ! *filefd = openat(p->rootfd, f->path, ! O_RDONLY | O_NOFOLLOW | O_NONBLOCK, 0); ! if (*filefd != -1 || errno == ENOENT) ! return 1; ! ERR("%s: openat", f->path); ! return -1; } /* --- 704,793 ---- return 0; } + if (sess->opts->max_size >= 0 && f->st.size > sess->opts->max_size) { + WARNX("skipping over max-size file %s", f->path); + return 0; + } + if (sess->opts->min_size >= 0 && f->st.size < sess->opts->min_size) { + WARNX("skipping under min-size file %s", f->path); + return 0; + } + /* * For non dry-run cases, we'll write the acknowledgement later ! * in the rsync_uploader() function. */ ! *size = 0; ! *filefd = -1; ! ! rc = check_file(p->rootfd, f, &st); ! if (rc == -1) ! return -1; ! if (rc == 2 && !S_ISREG(st.st_mode)) { ! if (S_ISDIR(st.st_mode) && ! unlinkat(p->rootfd, f->path, AT_REMOVEDIR) == -1) { ! ERR("%s: unlinkat", f->path); ! return -1; ! } ! } ! if (rc == 0) { ! if (!rsync_set_metadata_at(sess, 0, p->rootfd, f, f->path)) { ! ERRX1("rsync_set_metadata"); ! return -1; ! } ! LOG3("%s: skipping: up to date", f->path); ! return 0; ! } ! ! /* check alternative locations for better match */ ! for (i = 0; sess->opts->basedir[i] != NULL; i++) { ! const char *root = sess->opts->basedir[i]; ! int dfd, x; ! ! dfd = openat(p->rootfd, root, O_RDONLY | O_DIRECTORY); ! if (dfd == -1) ! err(ERR_FILE_IO, "%s: openat", root); ! x = check_file(dfd, f, &st); ! /* found a match */ ! if (x == 0) { ! if (rc >= 0) { ! /* found better match, delete file in rootfd */ ! if (unlinkat(p->rootfd, f->path, 0) == -1 && ! errno != ENOENT) { ! ERR("%s: unlinkat", f->path); ! return -1; ! } ! } ! LOG3("%s: skipping: up to date in %s", f->path, root); ! /* TODO: depending on mode link or copy file */ ! close(dfd); ! return 0; ! } else if (x == 1 && match == -1) { ! /* found a local file that is a close match */ ! match = i; ! } ! close(dfd); ! } ! if (match != -1) { ! /* copy match from basedir into root as a start point */ ! copy_file(p->rootfd, sess->opts->basedir[match], f); ! if (fstatat(p->rootfd, f->path, &st, AT_SYMLINK_NOFOLLOW) == ! -1) { ! ERR("%s: fstatat", f->path); ! return -1; ! } ! } ! ! *size = st.st_size; ! *filefd = openat(p->rootfd, f->path, O_RDONLY | O_NOFOLLOW); ! if (*filefd == -1 && errno != ENOENT) { ! ERR("%s: openat", f->path); ! return -1; ! } ! ! /* file needs attention */ ! return 1; } /* *************** *** 737,752 **** struct sess *sess, int *fileoutfd) { struct blkset blk; - struct stat st; void *mbuf, *bufp; ssize_t msz; size_t i, pos, sz; ! off_t offs; int c; - const struct flist *f; ! /* This should never get called. */ ! assert(u->state != UPLOAD_FINISHED); /* --- 858,870 ---- struct sess *sess, int *fileoutfd) { struct blkset blk; void *mbuf, *bufp; ssize_t msz; size_t i, pos, sz; ! off_t offs, filesize; int c; ! /* Once finished this should never get called again. */ assert(u->state != UPLOAD_FINISHED); /* *************** *** 756,762 **** * have a valid buffer to write. */ ! if (u->state == UPLOAD_WRITE_LOCAL) { assert(u->buf != NULL); assert(*fileoutfd != -1); assert(*fileinfd == -1); --- 874,880 ---- * have a valid buffer to write. */ ! if (u->state == UPLOAD_WRITE) { assert(u->buf != NULL); assert(*fileoutfd != -1); assert(*fileinfd == -1); *************** *** 810,818 **** if (S_ISDIR(u->fl[u->idx].st.mode)) c = pre_dir(u, sess); else if (S_ISLNK(u->fl[u->idx].st.mode)) ! c = pre_link(u, sess); else if (S_ISREG(u->fl[u->idx].st.mode)) ! c = pre_file(u, fileinfd, sess); else if (S_ISBLK(u->fl[u->idx].st.mode) || S_ISCHR(u->fl[u->idx].st.mode)) c = pre_dev(u, sess); --- 928,936 ---- if (S_ISDIR(u->fl[u->idx].st.mode)) c = pre_dir(u, sess); else if (S_ISLNK(u->fl[u->idx].st.mode)) ! c = pre_symlink(u, sess); else if (S_ISREG(u->fl[u->idx].st.mode)) ! c = pre_file(u, fileinfd, &filesize, sess); else if (S_ISBLK(u->fl[u->idx].st.mode) || S_ISCHR(u->fl[u->idx].st.mode)) c = pre_dev(u, sess); *************** *** 848,913 **** /* Go back to the event loop, if necessary. */ ! u->state = (*fileinfd == -1) ? ! UPLOAD_WRITE_LOCAL : UPLOAD_READ_LOCAL; ! if (u->state == UPLOAD_READ_LOCAL) ! return 1; } - /* - * If an input file is open, stat it and see if it's already up - * to date, in which case close it and go to the next one. - * Either way, we don't have a write channel open. - */ - - if (u->state == UPLOAD_READ_LOCAL) { - assert(*fileinfd != -1); - assert(*fileoutfd == -1); - f = &u->fl[u->idx]; - - if (fstat(*fileinfd, &st) == -1) { - ERR("%s: fstat", f->path); - close(*fileinfd); - *fileinfd = -1; - return -1; - } else if (!S_ISREG(st.st_mode)) { - ERRX("%s: not regular", f->path); - close(*fileinfd); - *fileinfd = -1; - return -1; - } - - if (st.st_size == f->st.size && - st.st_mtime == f->st.mtime) { - LOG3("%s: skipping: up to date", f->path); - if (!rsync_set_metadata - (sess, 0, *fileinfd, f, f->path)) { - ERRX1("rsync_set_metadata"); - close(*fileinfd); - *fileinfd = -1; - return -1; - } - close(*fileinfd); - *fileinfd = -1; - *fileoutfd = u->fdout; - u->state = UPLOAD_FIND_NEXT; - u->idx++; - return 1; - } - - /* Fallthrough... */ - - u->state = UPLOAD_WRITE_LOCAL; - } - /* Initialies our blocks. */ ! assert(u->state == UPLOAD_WRITE_LOCAL); memset(&blk, 0, sizeof(struct blkset)); blk.csum = u->csumlen; ! if (*fileinfd != -1 && st.st_size > 0) { ! init_blkset(&blk, st.st_size); assert(blk.blksz); blk.blks = calloc(blk.blksz, sizeof(struct blk)); --- 966,982 ---- /* Go back to the event loop, if necessary. */ ! u->state = UPLOAD_WRITE; } /* Initialies our blocks. */ ! assert(u->state == UPLOAD_WRITE); memset(&blk, 0, sizeof(struct blkset)); blk.csum = u->csumlen; ! if (*fileinfd != -1 && filesize > 0) { ! init_blkset(&blk, filesize); assert(blk.blksz); blk.blks = calloc(blk.blksz, sizeof(struct blk)); *************** *** 918,927 **** return -1; } ! if ((mbuf = calloc(1, blk.len)) == NULL) { ! ERR("calloc"); close(*fileinfd); *fileinfd = -1; return -1; } --- 987,997 ---- return -1; } ! if ((mbuf = malloc(blk.len)) == NULL) { ! ERR("malloc"); close(*fileinfd); *fileinfd = -1; + free(blk.blks); return -1; } *************** *** 929,944 **** i = 0; do { msz = pread(*fileinfd, mbuf, blk.len, offs); ! if (msz < 0) { ERR("pread"); close(*fileinfd); *fileinfd = -1; return -1; } - if ((size_t)msz != blk.len && (size_t)msz != blk.rem) { - /* short read, try again */ - continue; - } init_blk(&blk.blks[i], &blk, offs, i, mbuf, sess); offs += blk.len; LOG3( --- 999,1012 ---- i = 0; do { msz = pread(*fileinfd, mbuf, blk.len, offs); ! if ((size_t)msz != blk.len && (size_t)msz != blk.rem) { ERR("pread"); close(*fileinfd); *fileinfd = -1; + free(mbuf); + free(blk.blks); return -1; } init_blk(&blk.blks[i], &blk, offs, i, mbuf, sess); offs += blk.len; LOG3( *************** *** 947,952 **** --- 1015,1021 ---- i++; } while (i < blk.blksz); + free(mbuf); close(*fileinfd); *fileinfd = -1; LOG3("%s: mapped %jd B with %zu blocks", *************** *** 966,983 **** /* Make sure the block metadata buffer is big enough. */ u->bufsz = ! sizeof(int32_t) + /* identifier */ ! sizeof(int32_t) + /* block count */ ! sizeof(int32_t) + /* block length */ ! sizeof(int32_t) + /* checksum length */ ! sizeof(int32_t) + /* block remainder */ ! blk.blksz * ! (sizeof(int32_t) + /* short checksum */ ! blk.csum); /* long checksum */ if (u->bufsz > u->bufmax) { if ((bufp = realloc(u->buf, u->bufsz)) == NULL) { ERR("realloc"); return -1; } u->buf = bufp; --- 1035,1053 ---- /* Make sure the block metadata buffer is big enough. */ u->bufsz = ! sizeof(int32_t) + /* identifier */ ! sizeof(int32_t) + /* block count */ ! sizeof(int32_t) + /* block length */ ! sizeof(int32_t) + /* checksum length */ ! sizeof(int32_t) + /* block remainder */ ! blk.blksz * ! (sizeof(int32_t) + /* short checksum */ ! blk.csum); /* long checksum */ if (u->bufsz > u->bufmax) { if ((bufp = realloc(u->buf, u->bufsz)) == NULL) { ERR("realloc"); + free(blk.blks); return -1; } u->buf = bufp;