From d0ee10b38c770e32c77fa7c53d496f5164732235 Mon Sep 17 00:00:00 2001 From: Daniel Gustafsson Date: Tue, 22 Sep 2020 13:22:10 +0200 Subject: [PATCH] Refactor pg_service.conf and pg_restore TOC file parsing v2 Lines in pg_service.conf were limited to 256 bytes, and while there haven't been reports of this being an issue it seems quite plausible that the day will come. The TOC file parsing in pg_restore was using a limit of 100 bytes which required juggling when lines exceeded 100 bytes (even though only the first few bytes are of interest). Refactor both of these to use StringInfo backed pg_get_line_append. This follows up on the work done in commits 784b1ba1a2 and 8f8154a50 to remove hardwired line lenghts in file parsing. Discussion: https://postgr.es/m/48A4FA71-524E-41B9-953A-FD04EF36E2E7@yesql.se --- src/bin/pg_dump/pg_backup_archiver.c | 42 +++++++--------- src/interfaces/libpq/fe-connect.c | 73 +++++++++++++--------------- 2 files changed, 51 insertions(+), 64 deletions(-) diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c index bc47578294..ac20999095 100644 --- a/src/bin/pg_dump/pg_backup_archiver.c +++ b/src/bin/pg_dump/pg_backup_archiver.c @@ -30,9 +30,11 @@ #include #endif +#include "common/string.h" #include "dumputils.h" #include "fe_utils/string_utils.h" #include "libpq/libpq-fs.h" +#include "lib/stringinfo.h" #include "parallel.h" #include "pg_backup_archiver.h" #include "pg_backup_db.h" @@ -1367,8 +1369,7 @@ SortTocFromFile(Archive *AHX) ArchiveHandle *AH = (ArchiveHandle *) AHX; RestoreOptions *ropt = AH->public.ropt; FILE *fh; - char buf[100]; - bool incomplete_line; + StringInfoData linebuf; /* Allocate space for the 'wanted' array, and init it */ ropt->idWanted = (bool *) pg_malloc0(sizeof(bool) * AH->maxDumpId); @@ -1378,45 +1379,33 @@ SortTocFromFile(Archive *AHX) if (!fh) fatal("could not open TOC file \"%s\": %m", ropt->tocFile); - incomplete_line = false; - while (fgets(buf, sizeof(buf), fh) != NULL) + initStringInfo(&linebuf); + + while (pg_get_line_append(fh, &linebuf)) { - bool prev_incomplete_line = incomplete_line; - int buflen; char *cmnt; char *endptr; DumpId id; TocEntry *te; - /* - * Some lines in the file might be longer than sizeof(buf). This is - * no problem, since we only care about the leading numeric ID which - * can be at most a few characters; but we have to skip continuation - * bufferloads when processing a long line. - */ - buflen = strlen(buf); - if (buflen > 0 && buf[buflen - 1] == '\n') - incomplete_line = false; - else - incomplete_line = true; - if (prev_incomplete_line) - continue; - /* Truncate line at comment, if any */ - cmnt = strchr(buf, ';'); + cmnt = strchr(linebuf.data, ';'); if (cmnt != NULL) + { cmnt[0] = '\0'; + linebuf.len = cmnt - linebuf.data; + } /* Ignore if all blank */ - if (strspn(buf, " \t\r\n") == strlen(buf)) + if (strspn(linebuf.data, " \t\r\n") == linebuf.len) continue; /* Get an ID, check it's valid and not already seen */ - id = strtol(buf, &endptr, 10); - if (endptr == buf || id <= 0 || id > AH->maxDumpId || + id = strtol(linebuf.data, &endptr, 10); + if (endptr == linebuf.data || id <= 0 || id > AH->maxDumpId || ropt->idWanted[id - 1]) { - pg_log_warning("line ignored: %s", buf); + pg_log_warning("line ignored: %s", linebuf.data); continue; } @@ -1441,8 +1430,11 @@ SortTocFromFile(Archive *AHX) * restored. */ _moveBefore(AH->toc, te); + resetStringInfo(&linebuf); } + pg_free(linebuf.data); + if (fclose(fh) != 0) fatal("could not close TOC file: %m"); } diff --git a/src/interfaces/libpq/fe-connect.c b/src/interfaces/libpq/fe-connect.c index 724076a310..04a2a1c24c 100644 --- a/src/interfaces/libpq/fe-connect.c +++ b/src/interfaces/libpq/fe-connect.c @@ -28,6 +28,7 @@ #include "fe-auth.h" #include "libpq-fe.h" #include "libpq-int.h" +#include "lib/stringinfo.h" #include "mb/pg_wchar.h" #include "pg_config_paths.h" #include "port/pg_bswap.h" @@ -5011,8 +5012,6 @@ ldapServiceLookup(const char *purl, PQconninfoOption *options, #endif /* USE_LDAP */ -#define MAXBUFSIZE 256 - /* * parseServiceInfo: if a service name has been given, look it up and absorb * connection options from it into *options. @@ -5100,10 +5099,11 @@ parseServiceFile(const char *serviceFile, bool *group_found) { int linenr = 0, - i; + i, + result = 0; FILE *f; - char buf[MAXBUFSIZE], - *line; + char *line; + StringInfoData linebuf; f = fopen(serviceFile, "r"); if (f == NULL) @@ -5113,26 +5113,17 @@ parseServiceFile(const char *serviceFile, return 1; } - while ((line = fgets(buf, sizeof(buf), f)) != NULL) - { - int len; + initStringInfo(&linebuf); + while (pg_get_line_append(f, &linebuf)) + { linenr++; - if (strlen(line) >= sizeof(buf) - 1) - { - fclose(f); - printfPQExpBuffer(errorMessage, - libpq_gettext("line %d too long in service file \"%s\"\n"), - linenr, - serviceFile); - return 2; - } - /* ignore whitespace at end of line, especially the newline */ - len = strlen(line); - while (len > 0 && isspace((unsigned char) line[len - 1])) - line[--len] = '\0'; + while (linebuf.len > 0 && isspace((unsigned char) linebuf.data[linebuf.len - 1])) + linebuf.data[--linebuf.len] = '\0'; + + line = linebuf.data; /* ignore leading whitespace too */ while (*line && isspace((unsigned char) line[0])) @@ -5140,17 +5131,17 @@ parseServiceFile(const char *serviceFile, /* ignore comments and empty lines */ if (line[0] == '\0' || line[0] == '#') + { + resetStringInfo(&linebuf); continue; + } /* Check for right groupname */ if (line[0] == '[') { + /* group info already read */ if (*group_found) - { - /* group info already read */ - fclose(f); - return 0; - } + goto exit; if (strncmp(line + 1, service, strlen(service)) == 0 && line[strlen(service) + 1] == ']') @@ -5178,13 +5169,13 @@ parseServiceFile(const char *serviceFile, switch (rc) { case 0: - fclose(f); - return 0; + goto exit; case 1: case 3: - fclose(f); - return 3; + result = 3; + goto exit; case 2: + resetStringInfo(&linebuf); continue; } } @@ -5198,8 +5189,8 @@ parseServiceFile(const char *serviceFile, libpq_gettext("syntax error in service file \"%s\", line %d\n"), serviceFile, linenr); - fclose(f); - return 3; + result = 3; + goto exit; } *val++ = '\0'; @@ -5209,8 +5200,8 @@ parseServiceFile(const char *serviceFile, libpq_gettext("nested service specifications not supported in service file \"%s\", line %d\n"), serviceFile, linenr); - fclose(f); - return 3; + result = 3; + goto exit; } /* @@ -5228,8 +5219,8 @@ parseServiceFile(const char *serviceFile, { printfPQExpBuffer(errorMessage, libpq_gettext("out of memory\n")); - fclose(f); - return 3; + result = 3; + goto exit; } found_keyword = true; break; @@ -5242,16 +5233,20 @@ parseServiceFile(const char *serviceFile, libpq_gettext("syntax error in service file \"%s\", line %d\n"), serviceFile, linenr); - fclose(f); - return 3; + result = 3; + goto exit; } } } + + resetStringInfo(&linebuf); } +exit: fclose(f); + pfree(linebuf.data); - return 0; + return result; } -- 2.21.1 (Apple Git-122.3)