From 7925d17874d2f173899caff7894d348560674069 Mon Sep 17 00:00:00 2001 From: Jakub Wartak Date: Fri, 5 Jan 2024 11:15:24 +0100 Subject: [PATCH v3 4/4] Add --clone and --copy-file-range copy strategies to pg_combinebackup using pg_copyfile_offload(). Discussion: https://www.postgresql.org/message-id/flat/CA%2BhUKGJvLLNQtzb%3DZWcTsYF8kv8cR_%3DH17CX-eL8qNixeC4DAw%40mail.gmail.com#ce606227e39df74c6b2abf80b8eab04a --- src/bin/pg_combinebackup/copy_file.c | 64 +----- src/bin/pg_combinebackup/copy_file.h | 5 +- src/bin/pg_combinebackup/pg_combinebackup.c | 230 ++++++++++---------- src/bin/pg_combinebackup/reconstruct.c | 111 ++++------ src/bin/pg_combinebackup/reconstruct.h | 22 +- 5 files changed, 184 insertions(+), 248 deletions(-) diff --git a/src/bin/pg_combinebackup/copy_file.c b/src/bin/pg_combinebackup/copy_file.c index 40a55e3087..448bfcd642 100644 --- a/src/bin/pg_combinebackup/copy_file.c +++ b/src/bin/pg_combinebackup/copy_file.c @@ -10,14 +10,12 @@ */ #include "postgres_fe.h" -#ifdef HAVE_COPYFILE_H -#include -#endif #include #include #include #include "common/file_perm.h" +#include "common/file_utils.h" #include "common/logging.h" #include "copy_file.h" @@ -35,7 +33,8 @@ static void copy_file_copyfile(const char *src, const char *dst); */ void copy_file(const char *src, const char *dst, - pg_checksum_context *checksum_ctx, bool dry_run) + pg_checksum_context *checksum_ctx, bool dry_run, + CopyFileMethod copy_strategy) { /* * In dry-run mode, we don't actually copy anything, nor do we read any @@ -49,6 +48,8 @@ copy_file(const char *src, const char *dst, pg_fatal("could not open \"%s\": %m", src); if (close(fd) < 0) pg_fatal("could not close \"%s\": %m", src); + + return; } /* @@ -56,56 +57,13 @@ copy_file(const char *src, const char *dst, * operating system primitives that we know about to copy the file; this * may be quicker than a naive block copy. */ - if (checksum_ctx->type == CHECKSUM_TYPE_NONE) - { - char *strategy_name = NULL; - void (*strategy_implementation) (const char *, const char *) = NULL; - -#ifdef WIN32 - strategy_name = "CopyFile"; - strategy_implementation = copy_file_copyfile; -#endif - - if (strategy_name != NULL) - { - if (dry_run) - pg_log_debug("would copy \"%s\" to \"%s\" using strategy %s", - src, dst, strategy_name); - else - { - pg_log_debug("copying \"%s\" to \"%s\" using strategy %s", - src, dst, strategy_name); - (*strategy_implementation) (src, dst); - } - return; - } - } - - /* - * Fall back to the simple approach of reading and writing all the blocks, - * feeding them into the checksum context as we go. - */ - if (dry_run) - { - if (checksum_ctx->type == CHECKSUM_TYPE_NONE) - pg_log_debug("would copy \"%s\" to \"%s\"", - src, dst); - else - pg_log_debug("would copy \"%s\" to \"%s\" and checksum with %s", - src, dst, pg_checksum_type_name(checksum_ctx->type)); - } + if (checksum_ctx->type == CHECKSUM_TYPE_NONE && copy_strategy != 0) + pg_copyfile_offload(src, dst, NULL, copy_strategy); else - { - if (checksum_ctx->type == CHECKSUM_TYPE_NONE) - pg_log_debug("copying \"%s\" to \"%s\"", - src, dst); - else - pg_log_debug("copying \"%s\" to \"%s\" and checksumming with %s", - src, dst, pg_checksum_type_name(checksum_ctx->type)); - copy_file_blocks(src, dst, checksum_ctx); - } + pg_copyfile(src, dst, NULL, checksum_ctx); } +#if 0 /* * Copy a file block by block, and optionally compute a checksum as we go. */ @@ -138,7 +96,8 @@ copy_file_blocks(const char *src, const char *dst, if (wb < 0) pg_fatal("could not write file \"%s\": %m", dst); else - pg_fatal("could not write file \"%s\": wrote only %d of %d bytes at offset %u", + pg_fatal("could not write file \"%s\": wrote only %d of %d bytes at " + "offset %u", dst, (int) wb, (int) rb, offset); } @@ -167,3 +126,4 @@ copy_file_copyfile(const char *src, const char *dst) } } #endif /* WIN32 */ +#endif diff --git a/src/bin/pg_combinebackup/copy_file.h b/src/bin/pg_combinebackup/copy_file.h index 031030bacb..cafaa0bc9c 100644 --- a/src/bin/pg_combinebackup/copy_file.h +++ b/src/bin/pg_combinebackup/copy_file.h @@ -11,9 +11,12 @@ #ifndef COPY_FILE_H #define COPY_FILE_H +#include "c.h" #include "common/checksum_helper.h" +#include "common/file_utils.h" extern void copy_file(const char *src, const char *dst, - pg_checksum_context *checksum_ctx, bool dry_run); + pg_checksum_context *checksum_ctx, bool dry_run, + CopyFileMethod copy_strategy); #endif /* COPY_FILE_H */ diff --git a/src/bin/pg_combinebackup/pg_combinebackup.c b/src/bin/pg_combinebackup/pg_combinebackup.c index cef4941d84..64dbc6c5fe 100644 --- a/src/bin/pg_combinebackup/pg_combinebackup.c +++ b/src/bin/pg_combinebackup/pg_combinebackup.c @@ -25,15 +25,15 @@ #include "common/logging.h" #include "copy_file.h" #include "fe_utils/option_utils.h" +#include "getopt_long.h" #include "lib/stringinfo.h" #include "load_manifest.h" -#include "getopt_long.h" #include "reconstruct.h" #include "write_manifest.h" /* Incremental file naming convention. */ -#define INCREMENTAL_PREFIX "INCREMENTAL." -#define INCREMENTAL_PREFIX_LENGTH (sizeof(INCREMENTAL_PREFIX) - 1) +#define INCREMENTAL_PREFIX "INCREMENTAL." +#define INCREMENTAL_PREFIX_LENGTH (sizeof(INCREMENTAL_PREFIX) - 1) /* * Tracking for directories that need to be removed, or have their contents @@ -69,6 +69,7 @@ typedef struct cb_options pg_checksum_type manifest_checksums; bool no_manifest; DataDirSyncMethod sync_method; + CopyFileMethod copy_method; } cb_options; /* @@ -98,15 +99,10 @@ static void cleanup_directories_atexit(void); static void create_output_directory(char *dirname, cb_options *opt); static void help(const char *progname); static bool parse_oid(char *s, Oid *result); -static void process_directory_recursively(Oid tsoid, - char *input_directory, - char *output_directory, - char *relative_path, - int n_prior_backups, - char **prior_backup_dirs, - manifest_data **manifests, - manifest_writer *mwriter, - cb_options *opt); +static void process_directory_recursively( + Oid tsoid, char *input_directory, char *output_directory, + char *relative_path, int n_prior_backups, char **prior_backup_dirs, + manifest_data **manifests, manifest_writer *mwriter, cb_options *opt); static int read_pg_version_file(char *directory); static void remember_to_cleanup_directory(char *target_path, bool rmtopdir); static void reset_directory_cleanup_list(void); @@ -129,8 +125,9 @@ main(int argc, char *argv[]) {"manifest-checksums", required_argument, NULL, 1}, {"no-manifest", no_argument, NULL, 2}, {"sync-method", required_argument, NULL, 3}, - {NULL, 0, NULL, 0} - }; + {"clone", no_argument, NULL, 4}, + {"copy-file-range", no_argument, NULL, 5}, + {NULL, 0, NULL, 0}}; const char *progname; char *last_input_dir; @@ -154,10 +151,11 @@ main(int argc, char *argv[]) memset(&opt, 0, sizeof(opt)); opt.manifest_checksums = CHECKSUM_TYPE_CRC32C; opt.sync_method = DATA_DIR_SYNC_METHOD_FSYNC; + opt.copy_method = 0; /* process command-line options */ - while ((c = getopt_long(argc, argv, "dnNPo:T:", - long_options, &optindex)) != -1) + while ((c = getopt_long(argc, argv, "dnNPo:T:", long_options, &optindex)) != + -1) { switch (c) { @@ -178,10 +176,8 @@ main(int argc, char *argv[]) add_tablespace_mapping(&opt, optarg); break; case 1: - if (!pg_checksum_parse_type(optarg, - &opt.manifest_checksums)) - pg_fatal("unrecognized checksum algorithm: \"%s\"", - optarg); + if (!pg_checksum_parse_type(optarg, &opt.manifest_checksums)) + pg_fatal("unrecognized checksum algorithm: \"%s\"", optarg); break; case 2: opt.no_manifest = true; @@ -190,6 +186,12 @@ main(int argc, char *argv[]) if (!parse_sync_method(optarg, &opt.sync_method)) exit(1); break; + case 4: + opt.copy_method = PG_COPYFILE_IOCTL_FICLONE; + break; + case 5: + opt.copy_method = PG_COPYFILE_COPY_FILE_RANGE; + break; default: /* getopt_long already emitted a complaint */ pg_log_error_hint("Try \"%s --help\" for more information.", progname); @@ -211,6 +213,14 @@ main(int argc, char *argv[]) if (opt.no_manifest) opt.manifest_checksums = CHECKSUM_TYPE_NONE; + /* + * We cannot provide file copy/clone offload in case when we need to + * calculate checksums + */ + if (opt.copy_method != 0 && opt.manifest_checksums != CHECKSUM_TYPE_NONE) + pg_fatal("unable to use accelerated copy when manifest checksums " + "are to be calculated. Use --no-manifest"); + /* Read the server version from the final backup. */ version = read_pg_version_file(argv[argc - 1]); @@ -263,7 +273,8 @@ main(int argc, char *argv[]) * won't have the WAL ranges for the resulting manifest. */ if (manifests[n_prior_backups] == NULL) - pg_fatal("can't generate a manifest because no manifest is available for the final input backup"); + pg_fatal("can't generate a manifest because no manifest is available for " + "the final input backup"); } else mwriter = NULL; @@ -275,15 +286,15 @@ main(int argc, char *argv[]) { pg_log_debug("generating \"%s/backup_label\"", opt.output); last_backup_label->cursor = 0; - write_backup_label(opt.output, last_backup_label, - opt.manifest_checksums, mwriter); + write_backup_label(opt.output, last_backup_label, opt.manifest_checksums, + mwriter); } /* Process everything that's not part of a user-defined tablespace. */ pg_log_debug("processing backup directory \"%s\"", last_input_dir); - process_directory_recursively(InvalidOid, last_input_dir, opt.output, - NULL, n_prior_backups, prior_backup_dirs, - manifests, mwriter, &opt); + process_directory_recursively(InvalidOid, last_input_dir, opt.output, NULL, + n_prior_backups, prior_backup_dirs, manifests, + mwriter, &opt); /* Process user-defined tablespaces. */ for (ts = tablespaces; ts != NULL; ts = ts->next) @@ -299,16 +310,15 @@ main(int argc, char *argv[]) { char linkpath[MAXPGPATH]; - snprintf(linkpath, MAXPGPATH, "%s/pg_tblspc/%u", opt.output, - ts->oid); + snprintf(linkpath, MAXPGPATH, "%s/pg_tblspc/%u", opt.output, ts->oid); if (opt.dry_run) pg_log_debug("would create symbolic link from \"%s\" to \"%s\"", linkpath, ts->new_dir); else { - pg_log_debug("creating symbolic link from \"%s\" to \"%s\"", - linkpath, ts->new_dir); + pg_log_debug("creating symbolic link from \"%s\" to \"%s\"", linkpath, + ts->new_dir); if (symlink(ts->new_dir, linkpath) != 0) pg_fatal("could not create symbolic link from \"%s\" to \"%s\": %m", linkpath, ts->new_dir); @@ -322,21 +332,19 @@ main(int argc, char *argv[]) { pg_log_debug("creating directory \"%s\"", ts->new_dir); if (pg_mkdir_p(ts->new_dir, pg_dir_create_mode) == -1) - pg_fatal("could not create directory \"%s\": %m", - ts->new_dir); + pg_fatal("could not create directory \"%s\": %m", ts->new_dir); } } /* OK, now handle the directory contents. */ - process_directory_recursively(ts->oid, ts->old_dir, ts->new_dir, - NULL, n_prior_backups, prior_backup_dirs, - manifests, mwriter, &opt); + process_directory_recursively(ts->oid, ts->old_dir, ts->new_dir, NULL, + n_prior_backups, prior_backup_dirs, manifests, + mwriter, &opt); } /* Finalize the backup_manifest, if we're generating one. */ if (mwriter != NULL) - finalize_manifest(mwriter, - manifests[n_prior_backups]->first_wal_range); + finalize_manifest(mwriter, manifests[n_prior_backups]->first_wal_range); /* fsync that output directory unless we've been told not to do so */ if (!opt.no_sync) @@ -392,7 +400,9 @@ add_tablespace_mapping(cb_options *opt, char *arg) *dst_ptr++ = *arg_ptr; } if (!tsmap->old_dir[0] || !tsmap->new_dir[0]) - pg_fatal("invalid tablespace mapping format \"%s\", must be \"OLDDIR=NEWDIR\"", arg); + pg_fatal( + "invalid tablespace mapping format \"%s\", must be \"OLDDIR=NEWDIR\"", + arg); /* * All tablespaces are created with absolute directories, so specifying a @@ -464,8 +474,8 @@ check_backup_label_files(int n_backups, char **backup_dirs) pg_fatal("could not close \"%s\": %m", pathbuf); /* Parse the file contents. */ - parse_backup_label(pathbuf, buf, &start_tli, &start_lsn, - &previous_tli, &previous_lsn); + parse_backup_label(pathbuf, buf, &start_tli, &start_lsn, &previous_tli, + &previous_lsn); /* * Sanity checks. @@ -476,18 +486,19 @@ check_backup_label_files(int n_backups, char **backup_dirs) * we don't have that information. */ if (i > 0 && previous_tli == 0) - pg_fatal("backup at \"%s\" is a full backup, but only the first backup should be a full backup", + pg_fatal("backup at \"%s\" is a full backup, but only the first backup " + "should be a full backup", backup_dirs[i]); if (i == 0 && previous_tli != 0) - pg_fatal("backup at \"%s\" is an incremental backup, but the first backup should be a full backup", + pg_fatal("backup at \"%s\" is an incremental backup, but the first " + "backup should be a full backup", backup_dirs[i]); if (i < n_backups - 1 && start_tli != check_tli) pg_fatal("backup at \"%s\" starts on timeline %u, but expected %u", backup_dirs[i], start_tli, check_tli); if (i < n_backups - 1 && start_lsn != check_lsn) pg_fatal("backup at \"%s\" starts at LSN %X/%X, but expected %X/%X", - backup_dirs[i], - LSN_FORMAT_ARGS(start_lsn), + backup_dirs[i], LSN_FORMAT_ARGS(start_lsn), LSN_FORMAT_ARGS(check_lsn)); check_tli = previous_tli; check_lsn = previous_lsn; @@ -542,8 +553,7 @@ check_control_files(int n_backups, char **backup_dirs) /* Can't interpret control file if not current version. */ if (control_file->pg_control_version != PG_CONTROL_VERSION) - pg_fatal("%s: unexpected control file version", - controlpath); + pg_fatal("%s: unexpected control file version", controlpath); /* System identifiers should all match. */ if (i == n_backups - 1) @@ -667,14 +677,23 @@ help(const char *progname) printf(_("\nOptions:\n")); printf(_(" -d, --debug generate lots of debugging output\n")); printf(_(" -n, --dry-run don't actually do anything\n")); - printf(_(" -N, --no-sync do not wait for changes to be written safely to disk\n")); + printf(_(" -N, --no-sync do not wait for changes to be written " + "safely to disk\n")); printf(_(" -o, --output output directory\n")); - printf(_(" -T, --tablespace-mapping=OLDDIR=NEWDIR\n" + printf(_( + " -T, --tablespace-mapping=OLDDIR=NEWDIR\n" " relocate tablespace in OLDDIR to NEWDIR\n")); - printf(_(" --manifest-checksums=SHA{224,256,384,512}|CRC32C|NONE\n" + printf( + _(" --manifest-checksums=SHA{224,256,384,512}|CRC32C|NONE\n" " use algorithm for manifest checksums\n")); - printf(_(" --no-manifest suppress generation of backup manifest\n")); - printf(_(" --sync-method=METHOD set method for syncing files to disk\n")); + printf(_( + " --no-manifest suppress generation of backup manifest\n")); + printf( + _(" --sync-method=METHOD set method for syncing files to disk\n")); + printf(_(" --clone clone (reflink) instead of copying " + "files\n")); + printf( + _(" --copy-file-range copy using copy_file_range() syscall\n")); printf(_(" -?, --help show this help, then exit\n")); printf(_("\nReport bugs to <%s>.\n"), PACKAGE_BUGREPORT); @@ -721,15 +740,10 @@ parse_oid(char *s, Oid *result) * the locations of those previous backups. */ static void -process_directory_recursively(Oid tsoid, - char *input_directory, - char *output_directory, - char *relative_path, - int n_prior_backups, - char **prior_backup_dirs, - manifest_data **manifests, - manifest_writer *mwriter, - cb_options *opt) +process_directory_recursively( + Oid tsoid, char *input_directory, char *output_directory, + char *relative_path, int n_prior_backups, char **prior_backup_dirs, + manifest_data **manifests, manifest_writer *mwriter, cb_options *opt) { char ifulldir[MAXPGPATH]; char ofulldir[MAXPGPATH]; @@ -782,13 +796,11 @@ process_directory_recursively(Oid tsoid, } else { - snprintf(ifulldir, MAXPGPATH, "%s/%s", input_directory, - relative_path); - snprintf(ofulldir, MAXPGPATH, "%s/%s", output_directory, - relative_path); + snprintf(ifulldir, MAXPGPATH, "%s/%s", input_directory, relative_path); + snprintf(ofulldir, MAXPGPATH, "%s/%s", output_directory, relative_path); if (OidIsValid(tsoid)) - snprintf(manifest_prefix, MAXPGPATH, "pg_tblspc/%u/%s/", - tsoid, relative_path); + snprintf(manifest_prefix, MAXPGPATH, "pg_tblspc/%u/%s/", tsoid, + relative_path); else snprintf(manifest_prefix, MAXPGPATH, "%s/", relative_path); } @@ -824,8 +836,7 @@ process_directory_recursively(Oid tsoid, pg_checksum_context checksum_ctx; /* Ignore "." and ".." entries. */ - if (strcmp(de->d_name, ".") == 0 || - strcmp(de->d_name, "..") == 0) + if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) continue; /* Construct input path. */ @@ -861,11 +872,9 @@ process_directory_recursively(Oid tsoid, de->d_name); /* And recurse. */ - process_directory_recursively(tsoid, - input_directory, output_directory, - new_relative_path, - n_prior_backups, prior_backup_dirs, - manifests, mwriter, opt); + process_directory_recursively(tsoid, input_directory, output_directory, + new_relative_path, n_prior_backups, + prior_backup_dirs, manifests, mwriter, opt); continue; } @@ -883,46 +892,37 @@ process_directory_recursively(Oid tsoid, * Skip the backup_label and backup_manifest files; they require * special handling and are handled elsewhere. */ - if (relative_path == NULL && - (strcmp(de->d_name, "backup_label") == 0 || - strcmp(de->d_name, "backup_manifest") == 0)) + if (relative_path == NULL && (strcmp(de->d_name, "backup_label") == 0 || + strcmp(de->d_name, "backup_manifest") == 0)) continue; /* * If it's an incremental file, hand it off to the reconstruction * code, which will figure out what to do. */ - if (strncmp(de->d_name, INCREMENTAL_PREFIX, - INCREMENTAL_PREFIX_LENGTH) == 0) + if (strncmp(de->d_name, INCREMENTAL_PREFIX, INCREMENTAL_PREFIX_LENGTH) == + 0) { /* Output path should not include "INCREMENTAL." prefix. */ snprintf(ofullpath, MAXPGPATH, "%s/%s", ofulldir, de->d_name + INCREMENTAL_PREFIX_LENGTH); - /* Manifest path likewise omits incremental prefix. */ snprintf(manifest_path, MAXPGPATH, "%s%s", manifest_prefix, de->d_name + INCREMENTAL_PREFIX_LENGTH); /* Reconstruction logic will do the rest. */ - reconstruct_from_incremental_file(ifullpath, ofullpath, - relative_path, - de->d_name + INCREMENTAL_PREFIX_LENGTH, - n_prior_backups, - prior_backup_dirs, - manifests, - manifest_path, - checksum_type, - &checksum_length, - &checksum_payload, - opt->debug, - opt->dry_run); + reconstruct_from_incremental_file( + ifullpath, ofullpath, relative_path, + de->d_name + INCREMENTAL_PREFIX_LENGTH, n_prior_backups, + prior_backup_dirs, manifests, manifest_path, checksum_type, + &checksum_length, &checksum_payload, opt->debug, opt->dry_run, + opt->copy_method); } else { /* Construct the path that the backup_manifest will use. */ - snprintf(manifest_path, MAXPGPATH, "%s%s", manifest_prefix, - de->d_name); + snprintf(manifest_path, MAXPGPATH, "%s%s", manifest_prefix, de->d_name); /* * It's not an incremental file, so we need to copy the entire @@ -932,13 +932,11 @@ process_directory_recursively(Oid tsoid, * backup_manifest for the final input directory, we can save some * work by reusing that checksum instead of computing a new one. */ - if (checksum_type != CHECKSUM_TYPE_NONE && - latest_manifest != NULL) + if (checksum_type != CHECKSUM_TYPE_NONE && latest_manifest != NULL) { manifest_file *mfile; - mfile = manifest_files_lookup(latest_manifest->files, - manifest_path); + mfile = manifest_files_lookup(latest_manifest->files, manifest_path); if (mfile == NULL) { char *bmpath; @@ -947,10 +945,9 @@ process_directory_recursively(Oid tsoid, * The directory is out of sync with the backup_manifest, * so emit a warning. */ - bmpath = psprintf("%s/%s", input_directory, - "backup_manifest"); - pg_log_warning("\"%s\" contains no entry for \"%s\"", - bmpath, manifest_path); + bmpath = psprintf("%s/%s", input_directory, "backup_manifest"); + pg_log_warning("\"%s\" contains no entry for \"%s\"", bmpath, + manifest_path); pfree(bmpath); } else if (mfile->checksum_type == checksum_type) @@ -972,7 +969,8 @@ process_directory_recursively(Oid tsoid, /* Actually copy the file. */ snprintf(ofullpath, MAXPGPATH, "%s/%s", ofulldir, de->d_name); - copy_file(ifullpath, ofullpath, &checksum_ctx, opt->dry_run); + copy_file(ifullpath, ofullpath, &checksum_ctx, opt->dry_run, + opt->copy_method); /* * If copy_file() performed a checksum calculation for us, then @@ -982,8 +980,7 @@ process_directory_recursively(Oid tsoid, if (checksum_ctx.type != CHECKSUM_TYPE_NONE && !opt->dry_run) { checksum_payload = pg_malloc(PG_CHECKSUM_MAX_LENGTH); - checksum_length = pg_checksum_final(&checksum_ctx, - checksum_payload); + checksum_length = pg_checksum_final(&checksum_ctx, checksum_payload); } } @@ -1009,10 +1006,8 @@ process_directory_recursively(Oid tsoid, pg_fatal("could not stat file \"%s\": %m", ofullpath); /* OK, now do the work. */ - add_file_to_manifest(mwriter, manifest_path, - sb.st_size, sb.st_mtime, - checksum_type, checksum_length, - checksum_payload); + add_file_to_manifest(mwriter, manifest_path, sb.st_size, sb.st_mtime, + checksum_type, checksum_length, checksum_payload); } /* Avoid leaking memory. */ @@ -1120,7 +1115,8 @@ reset_directory_cleanup_list(void) * final backup in the backup chain. */ static cb_tablespace * -scan_for_existing_tablespaces(char *pathname, cb_options *opt) +scan_for_existing_tablespaces(char *pathname, + cb_options *opt) { char pg_tblspc[MAXPGPATH]; DIR *dir; @@ -1153,7 +1149,8 @@ scan_for_existing_tablespaces(char *pathname, cb_options *opt) /* Ignore any file name that doesn't look like a proper OID. */ if (!parse_oid(de->d_name, &oid)) { - pg_log_debug("skipping \"%s\" because the filename is not a legal tablespace OID", + pg_log_debug( + "skipping \"%s\" because the filename is not a legal tablespace OID", tblspcdir); continue; } @@ -1164,7 +1161,8 @@ scan_for_existing_tablespaces(char *pathname, cb_options *opt) exit(1); if (type != PGFILETYPE_LNK && type != PGFILETYPE_DIR) { - pg_log_debug("skipping \"%s\" because it is neither a symbolic link nor a directory", + pg_log_debug("skipping \"%s\" because it is neither a symbolic link nor " + "a directory", tblspcdir); continue; } @@ -1184,8 +1182,7 @@ scan_for_existing_tablespaces(char *pathname, cb_options *opt) /* Read the link target. */ link_length = readlink(tblspcdir, link_target, sizeof(link_target)); if (link_length < 0) - pg_fatal("could not read symbolic link \"%s\": %m", - tblspcdir); + pg_fatal("could not read symbolic link \"%s\": %m", tblspcdir); if (link_length >= sizeof(link_target)) pg_fatal("symbolic link \"%s\" is too long", tblspcdir); link_target[link_length] = '\0'; @@ -1212,8 +1209,7 @@ scan_for_existing_tablespaces(char *pathname, cb_options *opt) /* Every non-in-place tablespace must be mapped. */ if (tsmap == NULL) - pg_fatal("tablespace at \"%s\" has no tablespace mapping", - link_target); + pg_fatal("tablespace at \"%s\" has no tablespace mapping", link_target); } else { @@ -1274,8 +1270,8 @@ slurp_file(int fd, char *filename, StringInfo buf, int maxlen) if (rb < 0) pg_fatal("could not read file \"%s\": %m", filename); else - pg_fatal("could not read file \"%s\": read only %d of %d bytes", - filename, (int) rb, (int) st.st_size); + pg_fatal("could not read file \"%s\": read only %d of %d bytes", filename, + (int) rb, (int) st.st_size); } /* Adjust buffer length for new data and restore trailing-\0 invariant */ diff --git a/src/bin/pg_combinebackup/reconstruct.c b/src/bin/pg_combinebackup/reconstruct.c index 874e6cd150..3f66eb9ad6 100644 --- a/src/bin/pg_combinebackup/reconstruct.c +++ b/src/bin/pg_combinebackup/reconstruct.c @@ -15,8 +15,8 @@ #include #include "backup/basebackup_incremental.h" -#include "common/logging.h" #include "common/file_perm.h" +#include "common/logging.h" #include "copy_file.h" #include "lib/stringinfo.h" #include "reconstruct.h" @@ -46,20 +46,16 @@ typedef struct rfile off_t highest_offset_read; } rfile; -static void debug_reconstruction(int n_source, - rfile **sources, - bool dry_run); +static void debug_reconstruction(int n_source, rfile **sources, bool dry_run); static unsigned find_reconstructed_block_length(rfile *s); static rfile *make_incremental_rfile(char *filename); static rfile *make_rfile(char *filename, bool missing_ok); static void write_reconstructed_file(char *input_filename, char *output_filename, - unsigned block_length, - rfile **sourcemap, + unsigned block_length, rfile **sourcemap, off_t *offsetmap, pg_checksum_context *checksum_ctx, - bool debug, - bool dry_run); + bool debug, bool dry_run); static void read_bytes(rfile *rf, void *buffer, unsigned length); /* @@ -78,19 +74,13 @@ static void read_bytes(rfile *rf, void *buffer, unsigned length); * an array of pathnames where those backups can be found. */ void -reconstruct_from_incremental_file(char *input_filename, - char *output_filename, - char *relative_path, - char *bare_file_name, - int n_prior_backups, - char **prior_backup_dirs, - manifest_data **manifests, - char *manifest_path, - pg_checksum_type checksum_type, - int *checksum_length, - uint8 **checksum_payload, - bool debug, - bool dry_run) +reconstruct_from_incremental_file( + char *input_filename, char *output_filename, char *relative_path, + char *bare_file_name, int n_prior_backups, char **prior_backup_dirs, + manifest_data **manifests, char *manifest_path, + pg_checksum_type checksum_type, int *checksum_length, + uint8 **checksum_payload, bool debug, bool dry_run, + CopyFileMethod copy_method) { rfile **source; rfile *latest_source = NULL; @@ -167,8 +157,8 @@ reconstruct_from_incremental_file(char *input_filename, * Look for the full file in the previous backup. If not found, then * look for an incremental file instead. */ - snprintf(source_filename, MAXPGPATH, "%s/%s/%s", - prior_backup_dirs[sidx], relative_path, bare_file_name); + snprintf(source_filename, MAXPGPATH, "%s/%s/%s", prior_backup_dirs[sidx], + relative_path, bare_file_name); if ((s = make_rfile(source_filename, true)) == NULL) { snprintf(source_filename, MAXPGPATH, "%s/%s/INCREMENTAL.%s", @@ -231,8 +221,7 @@ reconstruct_from_incremental_file(char *input_filename, { uint64 expected_length; - expected_length = - (uint64) latest_source->truncation_block_length; + expected_length = (uint64) latest_source->truncation_block_length; expected_length *= BLCKSZ; if (expected_length == sb.st_size) { @@ -253,8 +242,7 @@ reconstruct_from_incremental_file(char *input_filename, { BlockNumber b = s->relative_block_numbers[i]; - if (b < latest_source->truncation_block_length && - sourcemap[b] == NULL) + if (b < latest_source->truncation_block_length && sourcemap[b] == NULL) { sourcemap[b] = s; offsetmap[b] = s->header_length + (i * BLCKSZ); @@ -283,16 +271,16 @@ reconstruct_from_incremental_file(char *input_filename, manifest_path); if (mfile == NULL) { - char *path = psprintf("%s/backup_manifest", - prior_backup_dirs[copy_source_index]); + char *path = + psprintf("%s/backup_manifest", prior_backup_dirs[copy_source_index]); /* * The directory is out of sync with the backup_manifest, so emit * a warning. */ - /*- translator: the first %s is a backup manifest file, the second is a file absent therein */ - pg_log_warning("\"%s\" contains no entry for \"%s\"", - path, + /*- translator: the first %s is a backup manifest file, the second is a + * file absent therein */ + pg_log_warning("\"%s\" contains no entry for \"%s\"", path, manifest_path); pfree(path); } @@ -300,8 +288,7 @@ reconstruct_from_incremental_file(char *input_filename, { *checksum_length = mfile->checksum_length; *checksum_payload = pg_malloc(*checksum_length); - memcpy(*checksum_payload, mfile->checksum_payload, - *checksum_length); + memcpy(*checksum_payload, mfile->checksum_payload, *checksum_length); checksum_type = CHECKSUM_TYPE_NONE; } } @@ -318,13 +305,13 @@ reconstruct_from_incremental_file(char *input_filename, * Otherwise, reconstruct. */ if (copy_source != NULL) - copy_file(copy_source->filename, output_filename, - &checksum_ctx, dry_run); + copy_file(copy_source->filename, output_filename, &checksum_ctx, dry_run, + copy_method); else { - write_reconstructed_file(input_filename, output_filename, - block_length, sourcemap, offsetmap, - &checksum_ctx, debug, dry_run); + write_reconstructed_file(input_filename, output_filename, block_length, + sourcemap, offsetmap, &checksum_ctx, debug, + dry_run); debug_reconstruction(n_prior_backups + 1, source, dry_run); } @@ -332,8 +319,7 @@ reconstruct_from_incremental_file(char *input_filename, if (checksum_type != CHECKSUM_TYPE_NONE) { *checksum_payload = pg_malloc(PG_CHECKSUM_MAX_LENGTH); - *checksum_length = pg_checksum_final(&checksum_ctx, - *checksum_payload); + *checksum_length = pg_checksum_final(&checksum_ctx, *checksum_payload); } /* @@ -378,11 +364,11 @@ debug_reconstruction(int n_source, rfile **sources, bool dry_run) /* Debug logging. */ if (dry_run) - pg_log_debug("would have read %u blocks from \"%s\"", - s->num_blocks_read, s->filename); + pg_log_debug("would have read %u blocks from \"%s\"", s->num_blocks_read, + s->filename); else - pg_log_debug("read %u blocks from \"%s\"", - s->num_blocks_read, s->filename); + pg_log_debug("read %u blocks from \"%s\"", s->num_blocks_read, + s->filename); /* * In dry-run mode, we don't actually try to read data from the file, @@ -401,8 +387,7 @@ debug_reconstruction(int n_source, rfile **sources, bool dry_run) pg_fatal("could not stat \"%s\": %m", s->filename); if (sb.st_size < s->highest_offset_read) pg_fatal("file \"%s\" is too short: expected %llu, found %llu", - s->filename, - (unsigned long long) s->highest_offset_read, + s->filename, (unsigned long long) s->highest_offset_read, (unsigned long long) sb.st_size); } } @@ -455,7 +440,8 @@ make_incremental_rfile(char *filename) read_bytes(rf, &rf->truncation_block_length, sizeof(rf->truncation_block_length)); if (rf->truncation_block_length > RELSEG_SIZE) - pg_fatal("file \"%s\" has truncation block length %u in excess of segment size %u", + pg_fatal("file \"%s\" has truncation block length %u in excess of segment " + "size %u", filename, rf->truncation_block_length, RELSEG_SIZE); /* Read block numbers if there are any. */ @@ -522,12 +508,10 @@ read_bytes(rfile *rf, void *buffer, unsigned length) static void write_reconstructed_file(char *input_filename, char *output_filename, - unsigned block_length, - rfile **sourcemap, + unsigned block_length, rfile **sourcemap, off_t *offsetmap, pg_checksum_context *checksum_ctx, - bool debug, - bool dry_run) + bool debug, bool dry_run) { int wfd = -1; unsigned i; @@ -570,14 +554,13 @@ write_reconstructed_file(char *input_filename, if (current_block == start_of_range) appendStringInfo(&debug_buf, " %u:zero", current_block); else - appendStringInfo(&debug_buf, " %u-%u:zero", - start_of_range, current_block); + appendStringInfo(&debug_buf, " %u-%u:zero", start_of_range, + current_block); } else { if (current_block == start_of_range) - appendStringInfo(&debug_buf, " %u:%s@" UINT64_FORMAT, - current_block, + appendStringInfo(&debug_buf, " %u:%s@" UINT64_FORMAT, current_block, s == NULL ? "ZERO" : s->filename, (uint64) offsetmap[current_block]); else @@ -604,8 +587,7 @@ write_reconstructed_file(char *input_filename, /* Open the output file, except in dry_run mode. */ if (!dry_run && - (wfd = open(output_filename, - O_RDWR | PG_BINARY | O_CREAT | O_EXCL, + (wfd = open(output_filename, O_RDWR | PG_BINARY | O_CREAT | O_EXCL, pg_file_create_mode)) < 0) pg_fatal("could not open file \"%s\": %m", output_filename); @@ -622,8 +604,8 @@ write_reconstructed_file(char *input_filename, else { s->num_blocks_read++; - s->highest_offset_read = Max(s->highest_offset_read, - offsetmap[i] + BLCKSZ); + s->highest_offset_read = + Max(s->highest_offset_read, offsetmap[i] + BLCKSZ); } /* Skip the rest of this in dry-run mode. */ @@ -650,9 +632,9 @@ write_reconstructed_file(char *input_filename, if (rb < 0) pg_fatal("could not read file \"%s\": %m", s->filename); else - pg_fatal("could not read file \"%s\": read only %d of %d bytes at offset %llu", - s->filename, rb, BLCKSZ, - (unsigned long long) offsetmap[i]); + pg_fatal("could not read file \"%s\": read only %d of %d bytes at " + "offset %llu", + s->filename, rb, BLCKSZ, (unsigned long long) offsetmap[i]); } } @@ -668,8 +650,7 @@ write_reconstructed_file(char *input_filename, /* Update the checksum computation. */ if (pg_checksum_update(checksum_ctx, buffer, BLCKSZ) < 0) - pg_fatal("could not update checksum of file \"%s\"", - output_filename); + pg_fatal("could not update checksum of file \"%s\"", output_filename); } /* Debugging output. */ diff --git a/src/bin/pg_combinebackup/reconstruct.h b/src/bin/pg_combinebackup/reconstruct.h index d689aeb5c2..6623ca932e 100644 --- a/src/bin/pg_combinebackup/reconstruct.h +++ b/src/bin/pg_combinebackup/reconstruct.h @@ -13,21 +13,17 @@ #ifndef RECONSTRUCT_H #define RECONSTRUCT_H +#include "c.h" #include "common/checksum_helper.h" +#include "common/file_utils.h" #include "load_manifest.h" -extern void reconstruct_from_incremental_file(char *input_filename, - char *output_filename, - char *relative_path, - char *bare_file_name, - int n_prior_backups, - char **prior_backup_dirs, - manifest_data **manifests, - char *manifest_path, - pg_checksum_type checksum_type, - int *checksum_length, - uint8 **checksum_payload, - bool debug, - bool dry_run); +extern void reconstruct_from_incremental_file( + char *input_filename, char *output_filename, char *relative_path, + char *bare_file_name, int n_prior_backups, char **prior_backup_dirs, + manifest_data **manifests, char *manifest_path, + pg_checksum_type checksum_type, int *checksum_length, + uint8 **checksum_payload, bool debug, bool dry_run, + CopyFileMethod copy_method); #endif -- 2.30.2