pgbench - refactor init functions with buffers

Started by Fabien COELHOabout 6 years ago30 messages
#1Fabien COELHO
coelho@cri.ensmp.fr
1 attachment(s)

Hello,

While developing pgbench to allow partitioned tabled, I reproduced the
string management style used in the corresponding functions, but was
pretty unhappy with that kind of pattern:

snprintf(buf + strlen(buf), sizeof(buf) - strlen(buf), ...)

However adding a feature is not the place for refactoring.

This patch refactors initialization functions so as to use PQExpBuffer
where appropriate to simplify and clarify the code. SQL commands are
generated by accumulating parts into a buffer in order, before executing
it. I also added a more generic function to execute a statement and fail
if the result is unexpected.

--
Fabien.

Attachments:

pgbench-buffer-1.patchtext/x-diff; name=pgbench-buffer-1.patchDownload
diff --git a/src/bin/pgbench/pgbench.c b/src/bin/pgbench/pgbench.c
index e72ad0036e..5450eba4ac 100644
--- a/src/bin/pgbench/pgbench.c
+++ b/src/bin/pgbench/pgbench.c
@@ -599,7 +599,7 @@ static void doLog(TState *thread, CState *st,
 				  StatsData *agg, bool skipped, double latency, double lag);
 static void processXactStats(TState *thread, CState *st, instr_time *now,
 							 bool skipped, StatsData *agg);
-static void append_fillfactor(char *opts, int len);
+static void append_fillfactor(PQExpBuffer query);
 static void addScript(ParsedScript script);
 static void *threadRun(void *arg);
 static void finishCon(CState *st);
@@ -1139,17 +1139,24 @@ accumStats(StatsData *stats, bool skipped, double lat, double lag)
 
 /* call PQexec() and exit() on failure */
 static void
+executeStatementExpect(PGconn *con, const char *sql, const ExecStatusType expected)
+{
+	PGresult   *res;
+
+	res = PQexec(con, sql);
+	if (PQresultStatus(res) != expected)
+	{
+		fprintf(stderr, "%s", PQerrorMessage(con));
+		exit(1);
+	}
+	PQclear(res);
+}
+
+/* execute sql command, which must work, or die if not */
+static void
 executeStatement(PGconn *con, const char *sql)
 {
-	PGresult   *res;
-
-	res = PQexec(con, sql);
-	if (PQresultStatus(res) != PGRES_COMMAND_OK)
-	{
-		fprintf(stderr, "%s", PQerrorMessage(con));
-		exit(1);
-	}
-	PQclear(res);
+	executeStatementExpect(con, sql, PGRES_COMMAND_OK);
 }
 
 /* call PQexec() and complain, but without exiting, on failure */
@@ -3631,30 +3638,26 @@ initDropTables(PGconn *con)
 static void
 createPartitions(PGconn *con)
 {
-	char		ff[64];
-
-	ff[0] = '\0';
-
-	/*
-	 * Per ddlinfo in initCreateTables, fillfactor is needed on table
-	 * pgbench_accounts.
-	 */
-	append_fillfactor(ff, sizeof(ff));
+	PQExpBufferData		query;
 
 	/* we must have to create some partitions */
 	Assert(partitions > 0);
 
 	fprintf(stderr, "creating %d partitions...\n", partitions);
 
+	initPQExpBuffer(&query);
+
 	for (int p = 1; p <= partitions; p++)
 	{
-		char		query[256];
-
 		if (partition_method == PART_RANGE)
 		{
 			int64		part_size = (naccounts * (int64) scale + partitions - 1) / partitions;
-			char		minvalue[32],
-						maxvalue[32];
+
+			printfPQExpBuffer(&query,
+							  "create%s table pgbench_accounts_%d\n"
+							  "  partition of pgbench_accounts\n"
+							  "  for values from (",
+							  unlogged_tables ? " unlogged" : "", p);
 
 			/*
 			 * For RANGE, we use open-ended partitions at the beginning and
@@ -3663,34 +3666,39 @@ createPartitions(PGconn *con)
 			 * scale, it is more generic and the performance is better.
 			 */
 			if (p == 1)
-				sprintf(minvalue, "minvalue");
+				appendPQExpBufferStr(&query, "minvalue");
 			else
-				sprintf(minvalue, INT64_FORMAT, (p - 1) * part_size + 1);
+				appendPQExpBuffer(&query, INT64_FORMAT, (p - 1) * part_size + 1);
+
+			appendPQExpBufferStr(&query, ") to (");
 
 			if (p < partitions)
-				sprintf(maxvalue, INT64_FORMAT, p * part_size + 1);
+				appendPQExpBuffer(&query, INT64_FORMAT, p * part_size + 1);
 			else
-				sprintf(maxvalue, "maxvalue");
+				appendPQExpBufferStr(&query, "maxvalue");
 
-			snprintf(query, sizeof(query),
-					 "create%s table pgbench_accounts_%d\n"
-					 "  partition of pgbench_accounts\n"
-					 "  for values from (%s) to (%s)%s\n",
-					 unlogged_tables ? " unlogged" : "", p,
-					 minvalue, maxvalue, ff);
+			appendPQExpBufferChar(&query, ')');
 		}
 		else if (partition_method == PART_HASH)
-			snprintf(query, sizeof(query),
-					 "create%s table pgbench_accounts_%d\n"
-					 "  partition of pgbench_accounts\n"
-					 "  for values with (modulus %d, remainder %d)%s\n",
-					 unlogged_tables ? " unlogged" : "", p,
-					 partitions, p - 1, ff);
+			printfPQExpBuffer(&query,
+							  "create%s table pgbench_accounts_%d\n"
+							  "  partition of pgbench_accounts\n"
+							  "  for values with (modulus %d, remainder %d)",
+							  unlogged_tables ? " unlogged" : "", p,
+							  partitions, p - 1);
 		else					/* cannot get there */
 			Assert(0);
 
-		executeStatement(con, query);
+		/*
+		 * Per ddlinfo in initCreateTables, fillfactor is needed on table
+		 * pgbench_accounts.
+		 */
+		append_fillfactor(&query);
+
+		executeStatement(con, query.data);
 	}
+
+	termPQExpBuffer(&query);
 }
 
 /*
@@ -3743,27 +3751,30 @@ initCreateTables(PGconn *con)
 			1
 		}
 	};
-	int			i;
+
+	PQExpBufferData		query;
 
 	fprintf(stderr, "creating tables...\n");
 
-	for (i = 0; i < lengthof(DDLs); i++)
+	initPQExpBuffer(&query);
+
+	for (int i = 0; i < lengthof(DDLs); i++)
 	{
-		char		opts[256];
-		char		buffer[256];
 		const struct ddlinfo *ddl = &DDLs[i];
-		const char *cols;
 
 		/* Construct new create table statement. */
-		opts[0] = '\0';
+		printfPQExpBuffer(&query, "create%s table %s(%s)",
+						  unlogged_tables ? " unlogged" : "",
+						  ddl->table,
+						  (scale >= SCALE_32BIT_THRESHOLD) ? ddl->bigcols : ddl->smcols);
 
 		/* Partition pgbench_accounts table */
 		if (partition_method != PART_NONE && strcmp(ddl->table, "pgbench_accounts") == 0)
-			snprintf(opts + strlen(opts), sizeof(opts) - strlen(opts),
-					 " partition by %s (aid)", PARTITION_METHOD[partition_method]);
+			appendPQExpBuffer(&query,
+							  " partition by %s (aid)", PARTITION_METHOD[partition_method]);
 		else if (ddl->declare_fillfactor)
 			/* fillfactor is only expected on actual tables */
-			append_fillfactor(opts, sizeof(opts));
+			append_fillfactor(&query);
 
 		if (tablespace != NULL)
 		{
@@ -3771,20 +3782,15 @@ initCreateTables(PGconn *con)
 
 			escape_tablespace = PQescapeIdentifier(con, tablespace,
 												   strlen(tablespace));
-			snprintf(opts + strlen(opts), sizeof(opts) - strlen(opts),
-					 " tablespace %s", escape_tablespace);
+			appendPQExpBuffer(&query, " tablespace %s", escape_tablespace);
 			PQfreemem(escape_tablespace);
 		}
 
-		cols = (scale >= SCALE_32BIT_THRESHOLD) ? ddl->bigcols : ddl->smcols;
-
-		snprintf(buffer, sizeof(buffer), "create%s table %s(%s)%s",
-				 unlogged_tables ? " unlogged" : "",
-				 ddl->table, cols, opts);
-
-		executeStatement(con, buffer);
+		executeStatement(con, query.data);
 	}
 
+	termPQExpBuffer(&query);
+
 	if (partition_method != PART_NONE)
 		createPartitions(con);
 }
@@ -3795,10 +3801,9 @@ initCreateTables(PGconn *con)
  * XXX - As default is 100, it could be removed in this case.
  */
 static void
-append_fillfactor(char *opts, int len)
+append_fillfactor(PQExpBuffer query)
 {
-	snprintf(opts + strlen(opts), len - strlen(opts),
-			 " with (fillfactor=%d)", fillfactor);
+	appendPQExpBuffer(query, " with (fillfactor=%d)", fillfactor);
 }
 
 /*
@@ -3807,10 +3812,7 @@ append_fillfactor(char *opts, int len)
 static void
 initGenerateData(PGconn *con)
 {
-	char		sql[256];
-	PGresult   *res;
-	int			i;
-	int64		k;
+	PQExpBufferData	sql;
 
 	/* used to track elapsed time and estimate of the remaining time */
 	instr_time	start,
@@ -3837,50 +3839,47 @@ initGenerateData(PGconn *con)
 					 "pgbench_history, "
 					 "pgbench_tellers");
 
+	initPQExpBuffer(&sql);
+
 	/*
 	 * fill branches, tellers, accounts in that order in case foreign keys
 	 * already exist
 	 */
-	for (i = 0; i < nbranches * scale; i++)
+	for (int i = 0; i < nbranches * scale; i++)
 	{
 		/* "filler" column defaults to NULL */
-		snprintf(sql, sizeof(sql),
-				 "insert into pgbench_branches(bid,bbalance) values(%d,0)",
-				 i + 1);
-		executeStatement(con, sql);
+		printfPQExpBuffer(&sql,
+						  "insert into pgbench_branches(bid,bbalance) values(%d,0)",
+						  i + 1);
+		executeStatement(con, sql.data);
 	}
 
-	for (i = 0; i < ntellers * scale; i++)
+	for (int i = 0; i < ntellers * scale; i++)
 	{
 		/* "filler" column defaults to NULL */
-		snprintf(sql, sizeof(sql),
-				 "insert into pgbench_tellers(tid,bid,tbalance) values (%d,%d,0)",
-				 i + 1, i / ntellers + 1);
-		executeStatement(con, sql);
+		printfPQExpBuffer(&sql,
+						  "insert into pgbench_tellers(tid,bid,tbalance) values (%d,%d,0)",
+						  i + 1, i / ntellers + 1);
+		executeStatement(con, sql.data);
 	}
 
 	/*
 	 * accounts is big enough to be worth using COPY and tracking runtime
 	 */
-	res = PQexec(con, "copy pgbench_accounts from stdin");
-	if (PQresultStatus(res) != PGRES_COPY_IN)
-	{
-		fprintf(stderr, "%s", PQerrorMessage(con));
-		exit(1);
-	}
-	PQclear(res);
+	executeStatementExpect(con, "copy pgbench_accounts from stdin", PGRES_COPY_IN);
 
 	INSTR_TIME_SET_CURRENT(start);
 
-	for (k = 0; k < (int64) naccounts * scale; k++)
+	/* printf overheads should probably be avoided... */
+	for (int64 k = 0; k < (int64) naccounts * scale; k++)
 	{
 		int64		j = k + 1;
 
 		/* "filler" column defaults to blank padded empty string */
-		snprintf(sql, sizeof(sql),
-				 INT64_FORMAT "\t" INT64_FORMAT "\t%d\t\n",
-				 j, k / naccounts + 1, 0);
-		if (PQputline(con, sql))
+		printfPQExpBuffer(&sql,
+						  INT64_FORMAT "\t" INT64_FORMAT "\t%d\t\n",
+						  j, k / naccounts + 1, 0);
+		if (PQputline(con, sql.data))
 		{
 			fprintf(stderr, "PQputline failed\n");
 			exit(1);
@@ -3936,6 +3935,8 @@ initGenerateData(PGconn *con)
 		exit(1);
 	}
 
+	termPQExpBuffer(&sql);
+
 	executeStatement(con, "commit");
 }
 
@@ -3963,14 +3964,15 @@ initCreatePKeys(PGconn *con)
 		"alter table pgbench_tellers add primary key (tid)",
 		"alter table pgbench_accounts add primary key (aid)"
 	};
-	int			i;
+	PQExpBufferData query;
 
 	fprintf(stderr, "creating primary keys...\n");
-	for (i = 0; i < lengthof(DDLINDEXes); i++)
-	{
-		char		buffer[256];
+	initPQExpBuffer(&query);
 
-		strlcpy(buffer, DDLINDEXes[i], sizeof(buffer));
+	for (int i = 0; i < lengthof(DDLINDEXes); i++)
+	{
+		resetPQExpBuffer(&query);
+		appendPQExpBufferStr(&query, DDLINDEXes[i]);
 
 		if (index_tablespace != NULL)
 		{
@@ -3978,13 +3980,15 @@ initCreatePKeys(PGconn *con)
 
 			escape_tablespace = PQescapeIdentifier(con, index_tablespace,
 												   strlen(index_tablespace));
-			snprintf(buffer + strlen(buffer), sizeof(buffer) - strlen(buffer),
-					 " using index tablespace %s", escape_tablespace);
+			appendPQExpBuffer(&query,
+							  " using index tablespace %s", escape_tablespace);
 			PQfreemem(escape_tablespace);
 		}
 
-		executeStatement(con, buffer);
+		executeStatement(con, query.data);
 	}
+
+	termPQExpBuffer(&query);
 }
 
 /*
#2Dilip Kumar
dilipbalaut@gmail.com
In reply to: Fabien COELHO (#1)
Re: pgbench - refactor init functions with buffers

On Tue, Oct 22, 2019 at 12:03 PM Fabien COELHO <coelho@cri.ensmp.fr> wrote:

Hello,

While developing pgbench to allow partitioned tabled, I reproduced the
string management style used in the corresponding functions, but was
pretty unhappy with that kind of pattern:

snprintf(buf + strlen(buf), sizeof(buf) - strlen(buf), ...)

However adding a feature is not the place for refactoring.

This patch refactors initialization functions so as to use PQExpBuffer
where appropriate to simplify and clarify the code. SQL commands are
generated by accumulating parts into a buffer in order, before executing
it. I also added a more generic function to execute a statement and fail
if the result is unexpected.

- for (i = 0; i < nbranches * scale; i++)
+ for (int i = 0; i < nbranches * scale; i++)
 ...
- for (i = 0; i < ntellers * scale; i++)
+ for (int i = 0; i < ntellers * scale; i++)
  {

I haven't read the complete patch. But, I have noticed that many
places you changed the variable declaration from c to c++ style (i.e
moved the declaration in the for loop). IMHO, generally in PG, we
don't follow this convention. Is there any specific reason to do
this?

--
Regards,
Dilip Kumar
EnterpriseDB: http://www.enterprisedb.com

#3Jeevan Ladhe
jeevan.ladhe@enterprisedb.com
In reply to: Dilip Kumar (#2)
Re: pgbench - refactor init functions with buffers

I haven't read the complete patch. But, I have noticed that many
places you changed the variable declaration from c to c++ style (i.e
moved the declaration in the for loop). IMHO, generally in PG, we
don't follow this convention. Is there any specific reason to do
this?

+1.

The patch does not apply on master, needs rebase.
Also, I got some whitespace errors.

I think you can also refactor the function tryExecuteStatement(), and
call your newly added function executeStatementExpect() by passing
an additional flag something like "errorOK".

Regards,
Jeevan Ladhe

#4Fabien COELHO
coelho@cri.ensmp.fr
In reply to: Dilip Kumar (#2)
Re: pgbench - refactor init functions with buffers

Hello Dilip,

- for (i = 0; i < nbranches * scale; i++)
+ for (int i = 0; i < nbranches * scale; i++)
...
- for (i = 0; i < ntellers * scale; i++)
+ for (int i = 0; i < ntellers * scale; i++)
{

I haven't read the complete patch. But, I have noticed that many
places you changed the variable declaration from c to c++ style (i.e
moved the declaration in the for loop). IMHO, generally in PG, we
don't follow this convention. Is there any specific reason to do
this?

There are many places where it is used now in pg (120 occurrences in
master, 7 in pgbench). I had a bug recently because of a stupidly reused
index variable, so I tend to use this now it is admissible, moreover here
I'm actually doing a refactoring patch, so it seems ok to include that.

--
Fabien.

#5Fabien COELHO
coelho@cri.ensmp.fr
In reply to: Jeevan Ladhe (#3)
1 attachment(s)
Re: pgbench - refactor init functions with buffers

Hello Jeevan,

I haven't read the complete patch. But, I have noticed that many
places you changed the variable declaration from c to c++ style (i.e
moved the declaration in the for loop). IMHO, generally in PG, we
don't follow this convention. Is there any specific reason to do
this?

+1.

As I said, this C99 feature is already used extensively in pg sources, so
it makes sense to use it when refactoring something and if appropriate,
which IMO is the case here.

The patch does not apply on master, needs rebase.

Hmmm. "git apply pgbench-buffer-1.patch" works for me on current master.

Also, I got some whitespace errors.

It possible, but I cannot see any. Could you be more specific?

Many mailers do not conform to MIME and mess-up newlines when attachements
are typed text/*, because MIME requires the mailer to convert those to
crnl eol when sending and back to system eol when receiving, but few
actually do it. Maybe the issue is really there.

I think you can also refactor the function tryExecuteStatement(), and
call your newly added function executeStatementExpect() by passing
an additional flag something like "errorOK".

Indeed, good point.

--
Fabien.

Attachments:

pgbench-buffer-2.patchtext/x-diff; name=pgbench-buffer-2.patchDownload
diff --git a/src/bin/pgbench/pgbench.c b/src/bin/pgbench/pgbench.c
index e72ad0036e..ef99016c23 100644
--- a/src/bin/pgbench/pgbench.c
+++ b/src/bin/pgbench/pgbench.c
@@ -599,7 +599,7 @@ static void doLog(TState *thread, CState *st,
 				  StatsData *agg, bool skipped, double latency, double lag);
 static void processXactStats(TState *thread, CState *st, instr_time *now,
 							 bool skipped, StatsData *agg);
-static void append_fillfactor(char *opts, int len);
+static void append_fillfactor(PQExpBuffer query);
 static void addScript(ParsedScript script);
 static void *threadRun(void *arg);
 static void finishCon(CState *st);
@@ -1137,34 +1137,36 @@ accumStats(StatsData *stats, bool skipped, double lat, double lag)
 	}
 }
 
-/* call PQexec() and exit() on failure */
+/* call PQexec() and possibly exit() on failure */
+static void
+executeStatementExpect(PGconn *con, const char *sql, const ExecStatusType expected, bool errorOK)
+{
+	PGresult   *res;
+
+	res = PQexec(con, sql);
+	if (PQresultStatus(res) != expected)
+	{
+		fprintf(stderr, "%s", PQerrorMessage(con));
+		if (errorOK)
+			fprintf(stderr, "(ignoring this error and continuing anyway)\n");
+		else
+			exit(1);
+	}
+	PQclear(res);
+}
+
+/* execute sql command, which must work, or die if not */
 static void
 executeStatement(PGconn *con, const char *sql)
 {
-	PGresult   *res;
-
-	res = PQexec(con, sql);
-	if (PQresultStatus(res) != PGRES_COMMAND_OK)
-	{
-		fprintf(stderr, "%s", PQerrorMessage(con));
-		exit(1);
-	}
-	PQclear(res);
+	executeStatementExpect(con, sql, PGRES_COMMAND_OK, false);
 }
 
 /* call PQexec() and complain, but without exiting, on failure */
 static void
 tryExecuteStatement(PGconn *con, const char *sql)
 {
-	PGresult   *res;
-
-	res = PQexec(con, sql);
-	if (PQresultStatus(res) != PGRES_COMMAND_OK)
-	{
-		fprintf(stderr, "%s", PQerrorMessage(con));
-		fprintf(stderr, "(ignoring this error and continuing anyway)\n");
-	}
-	PQclear(res);
+	executeStatementExpect(con, sql, PGRES_COMMAND_OK, true);
 }
 
 /* set up a connection to the backend */
@@ -3631,30 +3633,26 @@ initDropTables(PGconn *con)
 static void
 createPartitions(PGconn *con)
 {
-	char		ff[64];
-
-	ff[0] = '\0';
-
-	/*
-	 * Per ddlinfo in initCreateTables, fillfactor is needed on table
-	 * pgbench_accounts.
-	 */
-	append_fillfactor(ff, sizeof(ff));
+	PQExpBufferData		query;
 
 	/* we must have to create some partitions */
 	Assert(partitions > 0);
 
 	fprintf(stderr, "creating %d partitions...\n", partitions);
 
+	initPQExpBuffer(&query);
+
 	for (int p = 1; p <= partitions; p++)
 	{
-		char		query[256];
-
 		if (partition_method == PART_RANGE)
 		{
 			int64		part_size = (naccounts * (int64) scale + partitions - 1) / partitions;
-			char		minvalue[32],
-						maxvalue[32];
+
+			printfPQExpBuffer(&query,
+							  "create%s table pgbench_accounts_%d\n"
+							  "  partition of pgbench_accounts\n"
+							  "  for values from (",
+							  unlogged_tables ? " unlogged" : "", p);
 
 			/*
 			 * For RANGE, we use open-ended partitions at the beginning and
@@ -3663,34 +3661,39 @@ createPartitions(PGconn *con)
 			 * scale, it is more generic and the performance is better.
 			 */
 			if (p == 1)
-				sprintf(minvalue, "minvalue");
+				appendPQExpBufferStr(&query, "minvalue");
 			else
-				sprintf(minvalue, INT64_FORMAT, (p - 1) * part_size + 1);
+				appendPQExpBuffer(&query, INT64_FORMAT, (p - 1) * part_size + 1);
+
+			appendPQExpBufferStr(&query, ") to (");
 
 			if (p < partitions)
-				sprintf(maxvalue, INT64_FORMAT, p * part_size + 1);
+				appendPQExpBuffer(&query, INT64_FORMAT, p * part_size + 1);
 			else
-				sprintf(maxvalue, "maxvalue");
+				appendPQExpBufferStr(&query, "maxvalue");
 
-			snprintf(query, sizeof(query),
-					 "create%s table pgbench_accounts_%d\n"
-					 "  partition of pgbench_accounts\n"
-					 "  for values from (%s) to (%s)%s\n",
-					 unlogged_tables ? " unlogged" : "", p,
-					 minvalue, maxvalue, ff);
+			appendPQExpBufferChar(&query, ')');
 		}
 		else if (partition_method == PART_HASH)
-			snprintf(query, sizeof(query),
-					 "create%s table pgbench_accounts_%d\n"
-					 "  partition of pgbench_accounts\n"
-					 "  for values with (modulus %d, remainder %d)%s\n",
-					 unlogged_tables ? " unlogged" : "", p,
-					 partitions, p - 1, ff);
+			printfPQExpBuffer(&query,
+							  "create%s table pgbench_accounts_%d\n"
+							  "  partition of pgbench_accounts\n"
+							  "  for values with (modulus %d, remainder %d)",
+							  unlogged_tables ? " unlogged" : "", p,
+							  partitions, p - 1);
 		else					/* cannot get there */
 			Assert(0);
 
-		executeStatement(con, query);
+		/*
+		 * Per ddlinfo in initCreateTables, fillfactor is needed on table
+		 * pgbench_accounts.
+		 */
+		append_fillfactor(&query);
+
+		executeStatement(con, query.data);
 	}
+
+	termPQExpBuffer(&query);
 }
 
 /*
@@ -3743,27 +3746,30 @@ initCreateTables(PGconn *con)
 			1
 		}
 	};
-	int			i;
+
+	PQExpBufferData		query;
 
 	fprintf(stderr, "creating tables...\n");
 
-	for (i = 0; i < lengthof(DDLs); i++)
+	initPQExpBuffer(&query);
+
+	for (int i = 0; i < lengthof(DDLs); i++)
 	{
-		char		opts[256];
-		char		buffer[256];
 		const struct ddlinfo *ddl = &DDLs[i];
-		const char *cols;
 
 		/* Construct new create table statement. */
-		opts[0] = '\0';
+		printfPQExpBuffer(&query, "create%s table %s(%s)",
+						  unlogged_tables ? " unlogged" : "",
+						  ddl->table,
+						  (scale >= SCALE_32BIT_THRESHOLD) ? ddl->bigcols : ddl->smcols);
 
 		/* Partition pgbench_accounts table */
 		if (partition_method != PART_NONE && strcmp(ddl->table, "pgbench_accounts") == 0)
-			snprintf(opts + strlen(opts), sizeof(opts) - strlen(opts),
-					 " partition by %s (aid)", PARTITION_METHOD[partition_method]);
+			appendPQExpBuffer(&query,
+							  " partition by %s (aid)", PARTITION_METHOD[partition_method]);
 		else if (ddl->declare_fillfactor)
 			/* fillfactor is only expected on actual tables */
-			append_fillfactor(opts, sizeof(opts));
+			append_fillfactor(&query);
 
 		if (tablespace != NULL)
 		{
@@ -3771,20 +3777,15 @@ initCreateTables(PGconn *con)
 
 			escape_tablespace = PQescapeIdentifier(con, tablespace,
 												   strlen(tablespace));
-			snprintf(opts + strlen(opts), sizeof(opts) - strlen(opts),
-					 " tablespace %s", escape_tablespace);
+			appendPQExpBuffer(&query, " tablespace %s", escape_tablespace);
 			PQfreemem(escape_tablespace);
 		}
 
-		cols = (scale >= SCALE_32BIT_THRESHOLD) ? ddl->bigcols : ddl->smcols;
-
-		snprintf(buffer, sizeof(buffer), "create%s table %s(%s)%s",
-				 unlogged_tables ? " unlogged" : "",
-				 ddl->table, cols, opts);
-
-		executeStatement(con, buffer);
+		executeStatement(con, query.data);
 	}
 
+	termPQExpBuffer(&query);
+
 	if (partition_method != PART_NONE)
 		createPartitions(con);
 }
@@ -3795,10 +3796,9 @@ initCreateTables(PGconn *con)
  * XXX - As default is 100, it could be removed in this case.
  */
 static void
-append_fillfactor(char *opts, int len)
+append_fillfactor(PQExpBuffer query)
 {
-	snprintf(opts + strlen(opts), len - strlen(opts),
-			 " with (fillfactor=%d)", fillfactor);
+	appendPQExpBuffer(query, " with (fillfactor=%d)", fillfactor);
 }
 
 /*
@@ -3807,10 +3807,7 @@ append_fillfactor(char *opts, int len)
 static void
 initGenerateData(PGconn *con)
 {
-	char		sql[256];
-	PGresult   *res;
-	int			i;
-	int64		k;
+	PQExpBufferData	sql;
 
 	/* used to track elapsed time and estimate of the remaining time */
 	instr_time	start,
@@ -3837,50 +3834,47 @@ initGenerateData(PGconn *con)
 					 "pgbench_history, "
 					 "pgbench_tellers");
 
+	initPQExpBuffer(&sql);
+
 	/*
 	 * fill branches, tellers, accounts in that order in case foreign keys
 	 * already exist
 	 */
-	for (i = 0; i < nbranches * scale; i++)
+	for (int i = 0; i < nbranches * scale; i++)
 	{
 		/* "filler" column defaults to NULL */
-		snprintf(sql, sizeof(sql),
-				 "insert into pgbench_branches(bid,bbalance) values(%d,0)",
-				 i + 1);
-		executeStatement(con, sql);
+		printfPQExpBuffer(&sql,
+						  "insert into pgbench_branches(bid,bbalance) values(%d,0)",
+						  i + 1);
+		executeStatement(con, sql.data);
 	}
 
-	for (i = 0; i < ntellers * scale; i++)
+	for (int i = 0; i < ntellers * scale; i++)
 	{
 		/* "filler" column defaults to NULL */
-		snprintf(sql, sizeof(sql),
-				 "insert into pgbench_tellers(tid,bid,tbalance) values (%d,%d,0)",
-				 i + 1, i / ntellers + 1);
-		executeStatement(con, sql);
+		printfPQExpBuffer(&sql,
+						  "insert into pgbench_tellers(tid,bid,tbalance) values (%d,%d,0)",
+						  i + 1, i / ntellers + 1);
+		executeStatement(con, sql.data);
 	}
 
 	/*
 	 * accounts is big enough to be worth using COPY and tracking runtime
 	 */
-	res = PQexec(con, "copy pgbench_accounts from stdin");
-	if (PQresultStatus(res) != PGRES_COPY_IN)
-	{
-		fprintf(stderr, "%s", PQerrorMessage(con));
-		exit(1);
-	}
-	PQclear(res);
+	executeStatementExpect(con, "copy pgbench_accounts from stdin", PGRES_COPY_IN, false);
 
 	INSTR_TIME_SET_CURRENT(start);
 
-	for (k = 0; k < (int64) naccounts * scale; k++)
+	/* printf overheads should probably be avoided... */
+	for (int64 k = 0; k < (int64) naccounts * scale; k++)
 	{
 		int64		j = k + 1;
 
 		/* "filler" column defaults to blank padded empty string */
-		snprintf(sql, sizeof(sql),
-				 INT64_FORMAT "\t" INT64_FORMAT "\t%d\t\n",
-				 j, k / naccounts + 1, 0);
-		if (PQputline(con, sql))
+		printfPQExpBuffer(&sql,
+						  INT64_FORMAT "\t" INT64_FORMAT "\t%d\t\n",
+						  j, k / naccounts + 1, 0);
+		if (PQputline(con, sql.data))
 		{
 			fprintf(stderr, "PQputline failed\n");
 			exit(1);
@@ -3936,6 +3930,8 @@ initGenerateData(PGconn *con)
 		exit(1);
 	}
 
+	termPQExpBuffer(&sql);
+
 	executeStatement(con, "commit");
 }
 
@@ -3963,14 +3959,15 @@ initCreatePKeys(PGconn *con)
 		"alter table pgbench_tellers add primary key (tid)",
 		"alter table pgbench_accounts add primary key (aid)"
 	};
-	int			i;
+	PQExpBufferData query;
 
 	fprintf(stderr, "creating primary keys...\n");
-	for (i = 0; i < lengthof(DDLINDEXes); i++)
-	{
-		char		buffer[256];
+	initPQExpBuffer(&query);
 
-		strlcpy(buffer, DDLINDEXes[i], sizeof(buffer));
+	for (int i = 0; i < lengthof(DDLINDEXes); i++)
+	{
+		resetPQExpBuffer(&query);
+		appendPQExpBufferStr(&query, DDLINDEXes[i]);
 
 		if (index_tablespace != NULL)
 		{
@@ -3978,13 +3975,15 @@ initCreatePKeys(PGconn *con)
 
 			escape_tablespace = PQescapeIdentifier(con, index_tablespace,
 												   strlen(index_tablespace));
-			snprintf(buffer + strlen(buffer), sizeof(buffer) - strlen(buffer),
-					 " using index tablespace %s", escape_tablespace);
+			appendPQExpBuffer(&query,
+							  " using index tablespace %s", escape_tablespace);
 			PQfreemem(escape_tablespace);
 		}
 
-		executeStatement(con, buffer);
+		executeStatement(con, query.data);
 	}
+
+	termPQExpBuffer(&query);
 }
 
 /*
#6Dilip Kumar
dilipbalaut@gmail.com
In reply to: Fabien COELHO (#4)
Re: pgbench - refactor init functions with buffers

On Tue, Oct 22, 2019 at 3:30 PM Fabien COELHO <coelho@cri.ensmp.fr> wrote:

Hello Dilip,

- for (i = 0; i < nbranches * scale; i++)
+ for (int i = 0; i < nbranches * scale; i++)
...
- for (i = 0; i < ntellers * scale; i++)
+ for (int i = 0; i < ntellers * scale; i++)
{

I haven't read the complete patch. But, I have noticed that many
places you changed the variable declaration from c to c++ style (i.e
moved the declaration in the for loop). IMHO, generally in PG, we
don't follow this convention. Is there any specific reason to do
this?

There are many places where it is used now in pg (120 occurrences in
master, 7 in pgbench). I had a bug recently because of a stupidly reused
index variable, so I tend to use this now it is admissible, moreover here
I'm actually doing a refactoring patch, so it seems ok to include that.

I see. I was under impression that we don't use this style in PG.
But, since we are already using this style other places so no
objection from my side for this particular point.
Sorry for the noise.

--
Regards,
Dilip Kumar
EnterpriseDB: http://www.enterprisedb.com

#7Jeevan Ladhe
jeevan.ladhe@enterprisedb.com
In reply to: Fabien COELHO (#5)
Re: pgbench - refactor init functions with buffers

On Tue, Oct 22, 2019 at 4:36 PM Fabien COELHO <coelho@cri.ensmp.fr> wrote:

Hello Jeevan,

I haven't read the complete patch. But, I have noticed that many
places you changed the variable declaration from c to c++ style (i.e
moved the declaration in the for loop). IMHO, generally in PG, we
don't follow this convention. Is there any specific reason to do
this?

+1.

As I said, this C99 feature is already used extensively in pg sources, so
it makes sense to use it when refactoring something and if appropriate,
which IMO is the case here.

Ok, no problem.

The patch does not apply on master, needs rebase.

Hmmm. "git apply pgbench-buffer-1.patch" works for me on current master.

Also, I got some whitespace errors.

It possible, but I cannot see any. Could you be more specific?

For me it failing, see below:

$ git log -1
commit ad4b7aeb84434c958e2df76fa69b68493a889e4a
Author: Peter Eisentraut <peter@eisentraut.org>
Date: Tue Oct 22 10:35:54 2019 +0200

Make command order in test more sensible

Through several updates, the CREATE USER command has been separated
from where the user is actually used in the test.

$ git apply pgbench-buffer-1.patch
pgbench-buffer-1.patch:10: trailing whitespace.
static void append_fillfactor(PQExpBuffer query);
pgbench-buffer-1.patch:18: trailing whitespace.
executeStatementExpect(PGconn *con, const char *sql, const ExecStatusType
expected)
pgbench-buffer-1.patch:19: trailing whitespace.
{
pgbench-buffer-1.patch:20: trailing whitespace.
PGresult *res;
pgbench-buffer-1.patch:21: trailing whitespace.

error: patch failed: src/bin/pgbench/pgbench.c:599
error: src/bin/pgbench/pgbench.c: patch does not apply

$

Regards,
Jeevan Ladhe

#8Fabien COELHO
coelho@cri.ensmp.fr
In reply to: Jeevan Ladhe (#7)
Re: pgbench - refactor init functions with buffers

The patch does not apply on master, needs rebase.

Hmmm. "git apply pgbench-buffer-1.patch" works for me on current master.

Also, I got some whitespace errors.

It possible, but I cannot see any. Could you be more specific?

For me it failing, see below:

$ git log -1
commit ad4b7aeb84434c958e2df76fa69b68493a889e4a

Same for me, but it works:

Switched to a new branch 'test'
sh> git apply ~/pgbench-buffer-2.patch
sh> git st
On branch test
Changes not staged for commit: ...
modified: src/bin/pgbench/pgbench.c

sh> file ~/pgbench-buffer-2.patch
.../pgbench-buffer-2.patch: unified diff output, ASCII text

sh> sha1sum ~/pgbench-buffer-2.patch
eab8167ef3ec5eca814c44b30e07ee5631914f07 ...

I suspect that your mailer did or did not do something with the
attachment. Maybe try with "patch -p1 < foo.patch" at the root.

--
Fabien.

#9Jeevan Ladhe
jeevan.ladhe@enterprisedb.com
In reply to: Fabien COELHO (#8)
Re: pgbench - refactor init functions with buffers

I am able to apply the v2 patch with "patch -p1 "

-----

+static void
+executeStatementExpect(PGconn *con, const char *sql, const ExecStatusType
expected, bool errorOK)
+{

I think some instances like this need 80 column alignment?

-----

in initCreatePKeys():
+ for (int i = 0; i < lengthof(DDLINDEXes); i++)
+ {
+ resetPQExpBuffer(&query);
+ appendPQExpBufferStr(&query, DDLINDEXes[i]);

I think you can simply use printfPQExpBuffer() for the first append,
similar to
what you have used in createPartitions(), which is a combination of both
reset
and append.

-----

The pgbench tap tests are also running fine.

Regards,
Jeevan Ladhe

On Tue, Oct 22, 2019 at 8:57 PM Fabien COELHO <coelho@cri.ensmp.fr> wrote:

Show quoted text

The patch does not apply on master, needs rebase.

Hmmm. "git apply pgbench-buffer-1.patch" works for me on current master.

Also, I got some whitespace errors.

It possible, but I cannot see any. Could you be more specific?

For me it failing, see below:

$ git log -1
commit ad4b7aeb84434c958e2df76fa69b68493a889e4a

Same for me, but it works:

Switched to a new branch 'test'
sh> git apply ~/pgbench-buffer-2.patch
sh> git st
On branch test
Changes not staged for commit: ...
modified: src/bin/pgbench/pgbench.c

sh> file ~/pgbench-buffer-2.patch
.../pgbench-buffer-2.patch: unified diff output, ASCII text

sh> sha1sum ~/pgbench-buffer-2.patch
eab8167ef3ec5eca814c44b30e07ee5631914f07 ...

I suspect that your mailer did or did not do something with the
attachment. Maybe try with "patch -p1 < foo.patch" at the root.

--
Fabien.

#10Fabien COELHO
coelho@cri.ensmp.fr
In reply to: Jeevan Ladhe (#9)
1 attachment(s)
Re: pgbench - refactor init functions with buffers

Hello Jeevan,

+static void
+executeStatementExpect(PGconn *con, const char *sql, const ExecStatusType
expected, bool errorOK)
+{

I think some instances like this need 80 column alignment?

Yep. Applying the pgindent is kind-of a pain, so I tend to do a reasonable
job by hand and rely on the next global pgindent to fix such things. I
shorten the line anyway.

+ resetPQExpBuffer(&query);
+ appendPQExpBufferStr(&query, DDLINDEXes[i]);

I think you can simply use printfPQExpBuffer() for the first append,
similar to what you have used in createPartitions(), which is a
combination of both reset and append.

It could, but it would mean switching to using a format which is not very
useful here as it uses the simpler append*Str variant.

While looking at it, I noticed the repeated tablespace addition just
afterwards, so I factored it out as well in a function.

Attached v3 shorten some lines and adds "append_tablespace".

--
Fabien.

Attachments:

pgbench-buffer-3.patchtext/x-diff; name=pgbench-buffer-3.patchDownload
diff --git a/src/bin/pgbench/pgbench.c b/src/bin/pgbench/pgbench.c
index e72ad0036e..f3aa6026b0 100644
--- a/src/bin/pgbench/pgbench.c
+++ b/src/bin/pgbench/pgbench.c
@@ -599,7 +599,9 @@ static void doLog(TState *thread, CState *st,
 				  StatsData *agg, bool skipped, double latency, double lag);
 static void processXactStats(TState *thread, CState *st, instr_time *now,
 							 bool skipped, StatsData *agg);
-static void append_fillfactor(char *opts, int len);
+static void append_fillfactor(PQExpBuffer query);
+static void append_tablespace(PGconn *con, PQExpBuffer query,
+							  const char *kind, const char *tablespace);
 static void addScript(ParsedScript script);
 static void *threadRun(void *arg);
 static void finishCon(CState *st);
@@ -1137,34 +1139,37 @@ accumStats(StatsData *stats, bool skipped, double lat, double lag)
 	}
 }
 
-/* call PQexec() and exit() on failure */
+/* call PQexec() and possibly exit() on failure */
+static void
+executeStatementExpect(PGconn *con, const char *sql,
+					   const ExecStatusType expected, bool errorOK)
+{
+	PGresult   *res;
+
+	res = PQexec(con, sql);
+	if (PQresultStatus(res) != expected)
+	{
+		fprintf(stderr, "%s", PQerrorMessage(con));
+		if (errorOK)
+			fprintf(stderr, "(ignoring this error and continuing anyway)\n");
+		else
+			exit(1);
+	}
+	PQclear(res);
+}
+
+/* execute sql command, which must work, or die if not */
 static void
 executeStatement(PGconn *con, const char *sql)
 {
-	PGresult   *res;
-
-	res = PQexec(con, sql);
-	if (PQresultStatus(res) != PGRES_COMMAND_OK)
-	{
-		fprintf(stderr, "%s", PQerrorMessage(con));
-		exit(1);
-	}
-	PQclear(res);
+	executeStatementExpect(con, sql, PGRES_COMMAND_OK, false);
 }
 
 /* call PQexec() and complain, but without exiting, on failure */
 static void
 tryExecuteStatement(PGconn *con, const char *sql)
 {
-	PGresult   *res;
-
-	res = PQexec(con, sql);
-	if (PQresultStatus(res) != PGRES_COMMAND_OK)
-	{
-		fprintf(stderr, "%s", PQerrorMessage(con));
-		fprintf(stderr, "(ignoring this error and continuing anyway)\n");
-	}
-	PQclear(res);
+	executeStatementExpect(con, sql, PGRES_COMMAND_OK, true);
 }
 
 /* set up a connection to the backend */
@@ -3631,30 +3636,26 @@ initDropTables(PGconn *con)
 static void
 createPartitions(PGconn *con)
 {
-	char		ff[64];
-
-	ff[0] = '\0';
-
-	/*
-	 * Per ddlinfo in initCreateTables, fillfactor is needed on table
-	 * pgbench_accounts.
-	 */
-	append_fillfactor(ff, sizeof(ff));
+	PQExpBufferData		query;
 
 	/* we must have to create some partitions */
 	Assert(partitions > 0);
 
 	fprintf(stderr, "creating %d partitions...\n", partitions);
 
+	initPQExpBuffer(&query);
+
 	for (int p = 1; p <= partitions; p++)
 	{
-		char		query[256];
-
 		if (partition_method == PART_RANGE)
 		{
 			int64		part_size = (naccounts * (int64) scale + partitions - 1) / partitions;
-			char		minvalue[32],
-						maxvalue[32];
+
+			printfPQExpBuffer(&query,
+							  "create%s table pgbench_accounts_%d\n"
+							  "  partition of pgbench_accounts\n"
+							  "  for values from (",
+							  unlogged_tables ? " unlogged" : "", p);
 
 			/*
 			 * For RANGE, we use open-ended partitions at the beginning and
@@ -3663,34 +3664,39 @@ createPartitions(PGconn *con)
 			 * scale, it is more generic and the performance is better.
 			 */
 			if (p == 1)
-				sprintf(minvalue, "minvalue");
+				appendPQExpBufferStr(&query, "minvalue");
 			else
-				sprintf(minvalue, INT64_FORMAT, (p - 1) * part_size + 1);
+				appendPQExpBuffer(&query, INT64_FORMAT, (p - 1) * part_size + 1);
+
+			appendPQExpBufferStr(&query, ") to (");
 
 			if (p < partitions)
-				sprintf(maxvalue, INT64_FORMAT, p * part_size + 1);
+				appendPQExpBuffer(&query, INT64_FORMAT, p * part_size + 1);
 			else
-				sprintf(maxvalue, "maxvalue");
+				appendPQExpBufferStr(&query, "maxvalue");
 
-			snprintf(query, sizeof(query),
-					 "create%s table pgbench_accounts_%d\n"
-					 "  partition of pgbench_accounts\n"
-					 "  for values from (%s) to (%s)%s\n",
-					 unlogged_tables ? " unlogged" : "", p,
-					 minvalue, maxvalue, ff);
+			appendPQExpBufferChar(&query, ')');
 		}
 		else if (partition_method == PART_HASH)
-			snprintf(query, sizeof(query),
-					 "create%s table pgbench_accounts_%d\n"
-					 "  partition of pgbench_accounts\n"
-					 "  for values with (modulus %d, remainder %d)%s\n",
-					 unlogged_tables ? " unlogged" : "", p,
-					 partitions, p - 1, ff);
+			printfPQExpBuffer(&query,
+							  "create%s table pgbench_accounts_%d\n"
+							  "  partition of pgbench_accounts\n"
+							  "  for values with (modulus %d, remainder %d)",
+							  unlogged_tables ? " unlogged" : "", p,
+							  partitions, p - 1);
 		else					/* cannot get there */
 			Assert(0);
 
-		executeStatement(con, query);
+		/*
+		 * Per ddlinfo in initCreateTables, fillfactor is needed on table
+		 * pgbench_accounts.
+		 */
+		append_fillfactor(&query);
+
+		executeStatement(con, query.data);
 	}
+
+	termPQExpBuffer(&query);
 }
 
 /*
@@ -3743,48 +3749,39 @@ initCreateTables(PGconn *con)
 			1
 		}
 	};
-	int			i;
+
+	PQExpBufferData		query;
 
 	fprintf(stderr, "creating tables...\n");
 
-	for (i = 0; i < lengthof(DDLs); i++)
+	initPQExpBuffer(&query);
+
+	for (int i = 0; i < lengthof(DDLs); i++)
 	{
-		char		opts[256];
-		char		buffer[256];
 		const struct ddlinfo *ddl = &DDLs[i];
-		const char *cols;
 
 		/* Construct new create table statement. */
-		opts[0] = '\0';
+		printfPQExpBuffer(&query, "create%s table %s(%s)",
+						  unlogged_tables ? " unlogged" : "",
+						  ddl->table,
+						  (scale >= SCALE_32BIT_THRESHOLD) ? ddl->bigcols : ddl->smcols);
 
 		/* Partition pgbench_accounts table */
 		if (partition_method != PART_NONE && strcmp(ddl->table, "pgbench_accounts") == 0)
-			snprintf(opts + strlen(opts), sizeof(opts) - strlen(opts),
-					 " partition by %s (aid)", PARTITION_METHOD[partition_method]);
+			appendPQExpBuffer(&query,
+							  " partition by %s (aid)", PARTITION_METHOD[partition_method]);
 		else if (ddl->declare_fillfactor)
 			/* fillfactor is only expected on actual tables */
-			append_fillfactor(opts, sizeof(opts));
+			append_fillfactor(&query);
 
 		if (tablespace != NULL)
-		{
-			char	   *escape_tablespace;
+			append_tablespace(con, &query, "", tablespace);
 
-			escape_tablespace = PQescapeIdentifier(con, tablespace,
-												   strlen(tablespace));
-			snprintf(opts + strlen(opts), sizeof(opts) - strlen(opts),
-					 " tablespace %s", escape_tablespace);
-			PQfreemem(escape_tablespace);
-		}
-
-		cols = (scale >= SCALE_32BIT_THRESHOLD) ? ddl->bigcols : ddl->smcols;
-
-		snprintf(buffer, sizeof(buffer), "create%s table %s(%s)%s",
-				 unlogged_tables ? " unlogged" : "",
-				 ddl->table, cols, opts);
-
-		executeStatement(con, buffer);
+		executeStatement(con, query.data);
 	}
 
+	termPQExpBuffer(&query);
+
 	if (partition_method != PART_NONE)
 		createPartitions(con);
 }
@@ -3795,10 +3792,23 @@ initCreateTables(PGconn *con)
  * XXX - As default is 100, it could be removed in this case.
  */
 static void
-append_fillfactor(char *opts, int len)
+append_fillfactor(PQExpBuffer query)
 {
-	snprintf(opts + strlen(opts), len - strlen(opts),
-			 " with (fillfactor=%d)", fillfactor);
+	appendPQExpBuffer(query, " with (fillfactor=%d)", fillfactor);
+}
+
+/*
+ * add tablespace option.
+ */
+static void
+append_tablespace(PGconn *con, PQExpBuffer query, const char *kind,
+				  const char *tablespace)
+{
+	char	   *escape_tablespace;
+
+	escape_tablespace = PQescapeIdentifier(con, tablespace, strlen(tablespace));
+	appendPQExpBuffer(query, "%s tablespace %s", kind, escape_tablespace);
+	PQfreemem(escape_tablespace);
 }
 
 /*
@@ -3807,10 +3817,7 @@ append_fillfactor(char *opts, int len)
 static void
 initGenerateData(PGconn *con)
 {
-	char		sql[256];
-	PGresult   *res;
-	int			i;
-	int64		k;
+	PQExpBufferData	sql;
 
 	/* used to track elapsed time and estimate of the remaining time */
 	instr_time	start,
@@ -3837,50 +3844,47 @@ initGenerateData(PGconn *con)
 					 "pgbench_history, "
 					 "pgbench_tellers");
 
+	initPQExpBuffer(&sql);
+
 	/*
 	 * fill branches, tellers, accounts in that order in case foreign keys
 	 * already exist
 	 */
-	for (i = 0; i < nbranches * scale; i++)
+	for (int i = 0; i < nbranches * scale; i++)
 	{
 		/* "filler" column defaults to NULL */
-		snprintf(sql, sizeof(sql),
-				 "insert into pgbench_branches(bid,bbalance) values(%d,0)",
-				 i + 1);
-		executeStatement(con, sql);
+		printfPQExpBuffer(&sql,
+						  "insert into pgbench_branches(bid,bbalance) values(%d,0)",
+						  i + 1);
+		executeStatement(con, sql.data);
 	}
 
-	for (i = 0; i < ntellers * scale; i++)
+	for (int i = 0; i < ntellers * scale; i++)
 	{
 		/* "filler" column defaults to NULL */
-		snprintf(sql, sizeof(sql),
-				 "insert into pgbench_tellers(tid,bid,tbalance) values (%d,%d,0)",
-				 i + 1, i / ntellers + 1);
-		executeStatement(con, sql);
+		printfPQExpBuffer(&sql,
+						  "insert into pgbench_tellers(tid,bid,tbalance) values (%d,%d,0)",
+						  i + 1, i / ntellers + 1);
+		executeStatement(con, sql.data);
 	}
 
 	/*
 	 * accounts is big enough to be worth using COPY and tracking runtime
 	 */
-	res = PQexec(con, "copy pgbench_accounts from stdin");
-	if (PQresultStatus(res) != PGRES_COPY_IN)
-	{
-		fprintf(stderr, "%s", PQerrorMessage(con));
-		exit(1);
-	}
-	PQclear(res);
+	executeStatementExpect(con, "copy pgbench_accounts from stdin", PGRES_COPY_IN, false);
 
 	INSTR_TIME_SET_CURRENT(start);
 
-	for (k = 0; k < (int64) naccounts * scale; k++)
+	/* printf overheads should probably be avoided... */
+	for (int64 k = 0; k < (int64) naccounts * scale; k++)
 	{
 		int64		j = k + 1;
 
 		/* "filler" column defaults to blank padded empty string */
-		snprintf(sql, sizeof(sql),
-				 INT64_FORMAT "\t" INT64_FORMAT "\t%d\t\n",
-				 j, k / naccounts + 1, 0);
-		if (PQputline(con, sql))
+		printfPQExpBuffer(&sql,
+						  INT64_FORMAT "\t" INT64_FORMAT "\t%d\t\n",
+						  j, k / naccounts + 1, 0);
+		if (PQputline(con, sql.data))
 		{
 			fprintf(stderr, "PQputline failed\n");
 			exit(1);
@@ -3936,6 +3940,8 @@ initGenerateData(PGconn *con)
 		exit(1);
 	}
 
+	termPQExpBuffer(&sql);
+
 	executeStatement(con, "commit");
 }
 
@@ -3963,28 +3969,23 @@ initCreatePKeys(PGconn *con)
 		"alter table pgbench_tellers add primary key (tid)",
 		"alter table pgbench_accounts add primary key (aid)"
 	};
-	int			i;
+	PQExpBufferData query;
 
 	fprintf(stderr, "creating primary keys...\n");
-	for (i = 0; i < lengthof(DDLINDEXes); i++)
-	{
-		char		buffer[256];
+	initPQExpBuffer(&query);
 
-		strlcpy(buffer, DDLINDEXes[i], sizeof(buffer));
+	for (int i = 0; i < lengthof(DDLINDEXes); i++)
+	{
+		resetPQExpBuffer(&query);
+		appendPQExpBufferStr(&query, DDLINDEXes[i]);
 
 		if (index_tablespace != NULL)
-		{
-			char	   *escape_tablespace;
+			append_tablespace(con, &query, " using index", index_tablespace);
 
-			escape_tablespace = PQescapeIdentifier(con, index_tablespace,
-												   strlen(index_tablespace));
-			snprintf(buffer + strlen(buffer), sizeof(buffer) - strlen(buffer),
-					 " using index tablespace %s", escape_tablespace);
-			PQfreemem(escape_tablespace);
-		}
-
-		executeStatement(con, buffer);
+		executeStatement(con, query.data);
 	}
+
+	termPQExpBuffer(&query);
 }
 
 /*
#11Andres Freund
andres@anarazel.de
In reply to: Fabien COELHO (#10)
Re: pgbench - refactor init functions with buffers

Hi,

On 2019-10-24 08:33:06 +0200, Fabien COELHO wrote:

Attached v3 shorten some lines and adds "append_tablespace".

I'd prefer not to expand the use of pqexpbuffer in more places, and
instead rather see this use StringInfo, now that's also available to
frontend programs.

Greetings,

Andres Freund

#12Fabien COELHO
coelho@cri.ensmp.fr
In reply to: Andres Freund (#11)
1 attachment(s)
Re: pgbench - refactor init functions with buffers

Hello Andres,

Attached v3 shorten some lines and adds "append_tablespace".

A v4 which just extends the patch to newly added 'G'.

I'd prefer not to expand the use of pqexpbuffer in more places, and
instead rather see this use StringInfo, now that's also available to
frontend programs.

Franckly, one or the other does not matter much to me.

However, pgbench already uses PQExpBuffer, it uses PsqlScanState which
also uses PQExpBuffer, and it intrinsically depends on libpq which
provides PQExpBuffer: ISTM that it makes sense to keep going there, unless
PQExpBuffer support is to be dropped.

Switching all usages would involve a significant effort and having both
PQExpBuffer and string_info used in the same file for the same purpose
would be confusing.

--
Fabien.

Attachments:

pgbench-buffer-4.patchtext/x-diff; name=pgbench-buffer-4.patchDownload
diff --git a/src/bin/pgbench/pgbench.c b/src/bin/pgbench/pgbench.c
index 14dbc4510c..f32134ac22 100644
--- a/src/bin/pgbench/pgbench.c
+++ b/src/bin/pgbench/pgbench.c
@@ -601,7 +601,9 @@ static void doLog(TState *thread, CState *st,
 				  StatsData *agg, bool skipped, double latency, double lag);
 static void processXactStats(TState *thread, CState *st, instr_time *now,
 							 bool skipped, StatsData *agg);
-static void append_fillfactor(char *opts, int len);
+static void append_fillfactor(PQExpBuffer query);
+static void append_tablespace(PGconn *con, PQExpBuffer query,
+							  const char *kind, const char *tablespace);
 static void addScript(ParsedScript script);
 static void *threadRun(void *arg);
 static void finishCon(CState *st);
@@ -1139,34 +1141,37 @@ accumStats(StatsData *stats, bool skipped, double lat, double lag)
 	}
 }
 
-/* call PQexec() and exit() on failure */
+/* call PQexec() and possibly exit() on failure */
+static void
+executeStatementExpect(PGconn *con, const char *sql,
+					   const ExecStatusType expected, bool errorOK)
+{
+	PGresult   *res;
+
+	res = PQexec(con, sql);
+	if (PQresultStatus(res) != expected)
+	{
+		fprintf(stderr, "%s", PQerrorMessage(con));
+		if (errorOK)
+			fprintf(stderr, "(ignoring this error and continuing anyway)\n");
+		else
+			exit(1);
+	}
+	PQclear(res);
+}
+
+/* execute sql command, which must work, or die if not */
 static void
 executeStatement(PGconn *con, const char *sql)
 {
-	PGresult   *res;
-
-	res = PQexec(con, sql);
-	if (PQresultStatus(res) != PGRES_COMMAND_OK)
-	{
-		fprintf(stderr, "%s", PQerrorMessage(con));
-		exit(1);
-	}
-	PQclear(res);
+	executeStatementExpect(con, sql, PGRES_COMMAND_OK, false);
 }
 
 /* call PQexec() and complain, but without exiting, on failure */
 static void
 tryExecuteStatement(PGconn *con, const char *sql)
 {
-	PGresult   *res;
-
-	res = PQexec(con, sql);
-	if (PQresultStatus(res) != PGRES_COMMAND_OK)
-	{
-		fprintf(stderr, "%s", PQerrorMessage(con));
-		fprintf(stderr, "(ignoring this error and continuing anyway)\n");
-	}
-	PQclear(res);
+	executeStatementExpect(con, sql, PGRES_COMMAND_OK, true);
 }
 
 /* set up a connection to the backend */
@@ -3633,30 +3638,26 @@ initDropTables(PGconn *con)
 static void
 createPartitions(PGconn *con)
 {
-	char		ff[64];
-
-	ff[0] = '\0';
-
-	/*
-	 * Per ddlinfo in initCreateTables, fillfactor is needed on table
-	 * pgbench_accounts.
-	 */
-	append_fillfactor(ff, sizeof(ff));
+	PQExpBufferData		query;
 
 	/* we must have to create some partitions */
 	Assert(partitions > 0);
 
 	fprintf(stderr, "creating %d partitions...\n", partitions);
 
+	initPQExpBuffer(&query);
+
 	for (int p = 1; p <= partitions; p++)
 	{
-		char		query[256];
-
 		if (partition_method == PART_RANGE)
 		{
 			int64		part_size = (naccounts * (int64) scale + partitions - 1) / partitions;
-			char		minvalue[32],
-						maxvalue[32];
+
+			printfPQExpBuffer(&query,
+							  "create%s table pgbench_accounts_%d\n"
+							  "  partition of pgbench_accounts\n"
+							  "  for values from (",
+							  unlogged_tables ? " unlogged" : "", p);
 
 			/*
 			 * For RANGE, we use open-ended partitions at the beginning and
@@ -3665,34 +3666,39 @@ createPartitions(PGconn *con)
 			 * scale, it is more generic and the performance is better.
 			 */
 			if (p == 1)
-				sprintf(minvalue, "minvalue");
+				appendPQExpBufferStr(&query, "minvalue");
 			else
-				sprintf(minvalue, INT64_FORMAT, (p - 1) * part_size + 1);
+				appendPQExpBuffer(&query, INT64_FORMAT, (p - 1) * part_size + 1);
+
+			appendPQExpBufferStr(&query, ") to (");
 
 			if (p < partitions)
-				sprintf(maxvalue, INT64_FORMAT, p * part_size + 1);
+				appendPQExpBuffer(&query, INT64_FORMAT, p * part_size + 1);
 			else
-				sprintf(maxvalue, "maxvalue");
+				appendPQExpBufferStr(&query, "maxvalue");
 
-			snprintf(query, sizeof(query),
-					 "create%s table pgbench_accounts_%d\n"
-					 "  partition of pgbench_accounts\n"
-					 "  for values from (%s) to (%s)%s\n",
-					 unlogged_tables ? " unlogged" : "", p,
-					 minvalue, maxvalue, ff);
+			appendPQExpBufferChar(&query, ')');
 		}
 		else if (partition_method == PART_HASH)
-			snprintf(query, sizeof(query),
-					 "create%s table pgbench_accounts_%d\n"
-					 "  partition of pgbench_accounts\n"
-					 "  for values with (modulus %d, remainder %d)%s\n",
-					 unlogged_tables ? " unlogged" : "", p,
-					 partitions, p - 1, ff);
+			printfPQExpBuffer(&query,
+							  "create%s table pgbench_accounts_%d\n"
+							  "  partition of pgbench_accounts\n"
+							  "  for values with (modulus %d, remainder %d)",
+							  unlogged_tables ? " unlogged" : "", p,
+							  partitions, p - 1);
 		else					/* cannot get there */
 			Assert(0);
 
-		executeStatement(con, query);
+		/*
+		 * Per ddlinfo in initCreateTables, fillfactor is needed on table
+		 * pgbench_accounts.
+		 */
+		append_fillfactor(&query);
+
+		executeStatement(con, query.data);
 	}
+
+	termPQExpBuffer(&query);
 }
 
 /*
@@ -3745,48 +3751,39 @@ initCreateTables(PGconn *con)
 			1
 		}
 	};
-	int			i;
+
+	PQExpBufferData		query;
 
 	fprintf(stderr, "creating tables...\n");
 
-	for (i = 0; i < lengthof(DDLs); i++)
+	initPQExpBuffer(&query);
+
+	for (int i = 0; i < lengthof(DDLs); i++)
 	{
-		char		opts[256];
-		char		buffer[256];
 		const struct ddlinfo *ddl = &DDLs[i];
-		const char *cols;
 
 		/* Construct new create table statement. */
-		opts[0] = '\0';
+		printfPQExpBuffer(&query, "create%s table %s(%s)",
+						  unlogged_tables ? " unlogged" : "",
+						  ddl->table,
+						  (scale >= SCALE_32BIT_THRESHOLD) ? ddl->bigcols : ddl->smcols);
 
 		/* Partition pgbench_accounts table */
 		if (partition_method != PART_NONE && strcmp(ddl->table, "pgbench_accounts") == 0)
-			snprintf(opts + strlen(opts), sizeof(opts) - strlen(opts),
-					 " partition by %s (aid)", PARTITION_METHOD[partition_method]);
+			appendPQExpBuffer(&query,
+							  " partition by %s (aid)", PARTITION_METHOD[partition_method]);
 		else if (ddl->declare_fillfactor)
 			/* fillfactor is only expected on actual tables */
-			append_fillfactor(opts, sizeof(opts));
+			append_fillfactor(&query);
 
 		if (tablespace != NULL)
-		{
-			char	   *escape_tablespace;
+			append_tablespace(con, &query, "", tablespace);
 
-			escape_tablespace = PQescapeIdentifier(con, tablespace,
-												   strlen(tablespace));
-			snprintf(opts + strlen(opts), sizeof(opts) - strlen(opts),
-					 " tablespace %s", escape_tablespace);
-			PQfreemem(escape_tablespace);
-		}
-
-		cols = (scale >= SCALE_32BIT_THRESHOLD) ? ddl->bigcols : ddl->smcols;
-
-		snprintf(buffer, sizeof(buffer), "create%s table %s(%s)%s",
-				 unlogged_tables ? " unlogged" : "",
-				 ddl->table, cols, opts);
-
-		executeStatement(con, buffer);
+		executeStatement(con, query.data);
 	}
 
+	termPQExpBuffer(&query);
+
 	if (partition_method != PART_NONE)
 		createPartitions(con);
 }
@@ -3797,10 +3794,23 @@ initCreateTables(PGconn *con)
  * XXX - As default is 100, it could be removed in this case.
  */
 static void
-append_fillfactor(char *opts, int len)
+append_fillfactor(PQExpBuffer query)
 {
-	snprintf(opts + strlen(opts), len - strlen(opts),
-			 " with (fillfactor=%d)", fillfactor);
+	appendPQExpBuffer(query, " with (fillfactor=%d)", fillfactor);
+}
+
+/*
+ * add tablespace option.
+ */
+static void
+append_tablespace(PGconn *con, PQExpBuffer query, const char *kind,
+				  const char *tablespace)
+{
+	char	   *escape_tablespace;
+
+	escape_tablespace = PQescapeIdentifier(con, tablespace, strlen(tablespace));
+	appendPQExpBuffer(query, "%s tablespace %s", kind, escape_tablespace);
+	PQfreemem(escape_tablespace);
 }
 
 /*
@@ -3822,10 +3832,7 @@ initTruncateTables(PGconn *con)
 static void
 initGenerateDataClientSide(PGconn *con)
 {
-	char		sql[256];
-	PGresult   *res;
-	int			i;
-	int64		k;
+	PQExpBufferData	sql;
 
 	/* used to track elapsed time and estimate of the remaining time */
 	instr_time	start,
@@ -3845,50 +3852,47 @@ initGenerateDataClientSide(PGconn *con)
 	/* truncate away any old data */
 	initTruncateTables(con);
 
+	initPQExpBuffer(&sql);
+
 	/*
 	 * fill branches, tellers, accounts in that order in case foreign keys
 	 * already exist
 	 */
-	for (i = 0; i < nbranches * scale; i++)
+	for (int i = 0; i < nbranches * scale; i++)
 	{
 		/* "filler" column defaults to NULL */
-		snprintf(sql, sizeof(sql),
-				 "insert into pgbench_branches(bid,bbalance) values(%d,0)",
-				 i + 1);
-		executeStatement(con, sql);
+		printfPQExpBuffer(&sql,
+						  "insert into pgbench_branches(bid,bbalance) values(%d,0)",
+						  i + 1);
+		executeStatement(con, sql.data);
 	}
 
-	for (i = 0; i < ntellers * scale; i++)
+	for (int i = 0; i < ntellers * scale; i++)
 	{
 		/* "filler" column defaults to NULL */
-		snprintf(sql, sizeof(sql),
-				 "insert into pgbench_tellers(tid,bid,tbalance) values (%d,%d,0)",
-				 i + 1, i / ntellers + 1);
-		executeStatement(con, sql);
+		printfPQExpBuffer(&sql,
+						  "insert into pgbench_tellers(tid,bid,tbalance) values (%d,%d,0)",
+						  i + 1, i / ntellers + 1);
+		executeStatement(con, sql.data);
 	}
 
 	/*
 	 * accounts is big enough to be worth using COPY and tracking runtime
 	 */
-	res = PQexec(con, "copy pgbench_accounts from stdin");
-	if (PQresultStatus(res) != PGRES_COPY_IN)
-	{
-		fprintf(stderr, "%s", PQerrorMessage(con));
-		exit(1);
-	}
-	PQclear(res);
+	executeStatementExpect(con, "copy pgbench_accounts from stdin", PGRES_COPY_IN, false);
 
 	INSTR_TIME_SET_CURRENT(start);
 
-	for (k = 0; k < (int64) naccounts * scale; k++)
+	/* printf overheads should probably be avoided... */
+	for (int64 k = 0; k < (int64) naccounts * scale; k++)
 	{
 		int64		j = k + 1;
 
 		/* "filler" column defaults to blank padded empty string */
-		snprintf(sql, sizeof(sql),
-				 INT64_FORMAT "\t" INT64_FORMAT "\t%d\t\n",
-				 j, k / naccounts + 1, 0);
-		if (PQputline(con, sql))
+		printfPQExpBuffer(&sql,
+						  INT64_FORMAT "\t" INT64_FORMAT "\t%d\t\n",
+						  j, k / naccounts + 1, 0);
+		if (PQputline(con, sql.data))
 		{
 			fprintf(stderr, "PQputline failed\n");
 			exit(1);
@@ -3944,6 +3948,8 @@ initGenerateDataClientSide(PGconn *con)
 		exit(1);
 	}
 
+	termPQExpBuffer(&sql);
+
 	executeStatement(con, "commit");
 }
 
@@ -3957,7 +3963,7 @@ initGenerateDataClientSide(PGconn *con)
 static void
 initGenerateDataServerSide(PGconn *con)
 {
-	char		sql[256];
+	PQExpBufferData	sql;
 
 	fprintf(stderr, "generating data (server-side)...\n");
 
@@ -3970,24 +3976,28 @@ initGenerateDataServerSide(PGconn *con)
 	/* truncate away any old data */
 	initTruncateTables(con);
 
-	snprintf(sql, sizeof(sql),
-			 "insert into pgbench_branches(bid,bbalance) "
-			 "select bid, 0 "
-			 "from generate_series(1, %d) as bid", nbranches * scale);
-	executeStatement(con, sql);
+	initPQExpBuffer(&sql);
 
-	snprintf(sql, sizeof(sql),
-			 "insert into pgbench_tellers(tid,bid,tbalance) "
-			 "select tid, (tid - 1) / %d + 1, 0 "
-			 "from generate_series(1, %d) as tid", ntellers, ntellers * scale);
-	executeStatement(con, sql);
+	printfPQExpBuffer(&sql,
+					  "insert into pgbench_branches(bid,bbalance) "
+					  "select bid, 0 "
+					  "from generate_series(1, %d) as bid", nbranches * scale);
+	executeStatement(con, sql.data);
 
-	snprintf(sql, sizeof(sql),
-			 "insert into pgbench_accounts(aid,bid,abalance,filler) "
-			 "select aid, (aid - 1) / %d + 1, 0, '' "
-			 "from generate_series(1, "INT64_FORMAT") as aid",
-			 naccounts, (int64) naccounts * scale);
-	executeStatement(con, sql);
+	printfPQExpBuffer(&sql,
+					  "insert into pgbench_tellers(tid,bid,tbalance) "
+					  "select tid, (tid - 1) / %d + 1, 0 "
+					  "from generate_series(1, %d) as tid", ntellers, ntellers * scale);
+	executeStatement(con, sql.data);
+
+	printfPQExpBuffer(&sql,
+					  "insert into pgbench_accounts(aid,bid,abalance,filler) "
+					  "select aid, (aid - 1) / %d + 1, 0, '' "
+					  "from generate_series(1, "INT64_FORMAT") as aid",
+					  naccounts, (int64) naccounts * scale);
+	executeStatement(con, sql.data);
+
+	termPQExpBuffer(&sql);
 
 	executeStatement(con, "commit");
 }
@@ -4016,28 +4026,23 @@ initCreatePKeys(PGconn *con)
 		"alter table pgbench_tellers add primary key (tid)",
 		"alter table pgbench_accounts add primary key (aid)"
 	};
-	int			i;
+	PQExpBufferData query;
 
 	fprintf(stderr, "creating primary keys...\n");
-	for (i = 0; i < lengthof(DDLINDEXes); i++)
-	{
-		char		buffer[256];
+	initPQExpBuffer(&query);
 
-		strlcpy(buffer, DDLINDEXes[i], sizeof(buffer));
+	for (int i = 0; i < lengthof(DDLINDEXes); i++)
+	{
+		resetPQExpBuffer(&query);
+		appendPQExpBufferStr(&query, DDLINDEXes[i]);
 
 		if (index_tablespace != NULL)
-		{
-			char	   *escape_tablespace;
+			append_tablespace(con, &query, " using index", index_tablespace);
 
-			escape_tablespace = PQescapeIdentifier(con, index_tablespace,
-												   strlen(index_tablespace));
-			snprintf(buffer + strlen(buffer), sizeof(buffer) - strlen(buffer),
-					 " using index tablespace %s", escape_tablespace);
-			PQfreemem(escape_tablespace);
-		}
-
-		executeStatement(con, buffer);
+		executeStatement(con, query.data);
 	}
+
+	termPQExpBuffer(&query);
 }
 
 /*
#13Fabien COELHO
coelho@cri.ensmp.fr
In reply to: Fabien COELHO (#12)
1 attachment(s)
Re: pgbench - refactor init functions with buffers

Attached v3 shorten some lines and adds "append_tablespace".

A v4 which just extends the patch to newly added 'G'.

v5 is a rebase after 30a3e772.

--
Fabien.

Attachments:

pgbench-buffer-5.patchtext/x-diff; name=pgbench-buffer-5.patchDownload
diff --git a/src/bin/pgbench/pgbench.c b/src/bin/pgbench/pgbench.c
index ee1134aea2..9666c644b3 100644
--- a/src/bin/pgbench/pgbench.c
+++ b/src/bin/pgbench/pgbench.c
@@ -599,7 +599,9 @@ static void doLog(TState *thread, CState *st,
 				  StatsData *agg, bool skipped, double latency, double lag);
 static void processXactStats(TState *thread, CState *st, instr_time *now,
 							 bool skipped, StatsData *agg);
-static void append_fillfactor(char *opts, int len);
+static void append_fillfactor(PQExpBuffer query);
+static void append_tablespace(PGconn *con, PQExpBuffer query,
+							  const char *kind, const char *tablespace);
 static void addScript(ParsedScript script);
 static void *threadRun(void *arg);
 static void finishCon(CState *st);
@@ -1133,35 +1135,38 @@ accumStats(StatsData *stats, bool skipped, double lat, double lag)
 	}
 }
 
-/* call PQexec() and exit() on failure */
+/* call PQexec() and possibly exit() on failure */
 static void
-executeStatement(PGconn *con, const char *sql)
+executeStatementExpect(PGconn *con, const char *sql,
+					   const ExecStatusType expected, bool errorOK)
 {
 	PGresult   *res;
 
 	res = PQexec(con, sql);
-	if (PQresultStatus(res) != PGRES_COMMAND_OK)
+	if (PQresultStatus(res) != expected)
 	{
 		pg_log_fatal("query failed: %s", PQerrorMessage(con));
 		pg_log_info("query was: %s", sql);
-		exit(1);
+		if (errorOK)
+			pg_log_info("(ignoring this error and continuing anyway)");
+		else
+			exit(1);
 	}
 	PQclear(res);
 }
 
+/* execute sql command, which must work, or die if not */
+static void
+executeStatement(PGconn *con, const char *sql)
+{
+	executeStatementExpect(con, sql, PGRES_COMMAND_OK, false);
+}
+
 /* call PQexec() and complain, but without exiting, on failure */
 static void
 tryExecuteStatement(PGconn *con, const char *sql)
 {
-	PGresult   *res;
-
-	res = PQexec(con, sql);
-	if (PQresultStatus(res) != PGRES_COMMAND_OK)
-	{
-		pg_log_error("%s", PQerrorMessage(con));
-		pg_log_info("(ignoring this error and continuing anyway)");
-	}
-	PQclear(res);
+	executeStatementExpect(con, sql, PGRES_COMMAND_OK, true);
 }
 
 /* set up a connection to the backend */
@@ -3600,30 +3605,26 @@ initDropTables(PGconn *con)
 static void
 createPartitions(PGconn *con)
 {
-	char		ff[64];
-
-	ff[0] = '\0';
-
-	/*
-	 * Per ddlinfo in initCreateTables, fillfactor is needed on table
-	 * pgbench_accounts.
-	 */
-	append_fillfactor(ff, sizeof(ff));
+	PQExpBufferData		query;
 
 	/* we must have to create some partitions */
 	Assert(partitions > 0);
 
 	fprintf(stderr, "creating %d partitions...\n", partitions);
 
+	initPQExpBuffer(&query);
+
 	for (int p = 1; p <= partitions; p++)
 	{
-		char		query[256];
-
 		if (partition_method == PART_RANGE)
 		{
 			int64		part_size = (naccounts * (int64) scale + partitions - 1) / partitions;
-			char		minvalue[32],
-						maxvalue[32];
+
+			printfPQExpBuffer(&query,
+							  "create%s table pgbench_accounts_%d\n"
+							  "  partition of pgbench_accounts\n"
+							  "  for values from (",
+							  unlogged_tables ? " unlogged" : "", p);
 
 			/*
 			 * For RANGE, we use open-ended partitions at the beginning and
@@ -3632,34 +3633,39 @@ createPartitions(PGconn *con)
 			 * scale, it is more generic and the performance is better.
 			 */
 			if (p == 1)
-				sprintf(minvalue, "minvalue");
+				appendPQExpBufferStr(&query, "minvalue");
 			else
-				sprintf(minvalue, INT64_FORMAT, (p - 1) * part_size + 1);
+				appendPQExpBuffer(&query, INT64_FORMAT, (p - 1) * part_size + 1);
+
+			appendPQExpBufferStr(&query, ") to (");
 
 			if (p < partitions)
-				sprintf(maxvalue, INT64_FORMAT, p * part_size + 1);
+				appendPQExpBuffer(&query, INT64_FORMAT, p * part_size + 1);
 			else
-				sprintf(maxvalue, "maxvalue");
+				appendPQExpBufferStr(&query, "maxvalue");
 
-			snprintf(query, sizeof(query),
-					 "create%s table pgbench_accounts_%d\n"
-					 "  partition of pgbench_accounts\n"
-					 "  for values from (%s) to (%s)%s\n",
-					 unlogged_tables ? " unlogged" : "", p,
-					 minvalue, maxvalue, ff);
+			appendPQExpBufferChar(&query, ')');
 		}
 		else if (partition_method == PART_HASH)
-			snprintf(query, sizeof(query),
-					 "create%s table pgbench_accounts_%d\n"
-					 "  partition of pgbench_accounts\n"
-					 "  for values with (modulus %d, remainder %d)%s\n",
-					 unlogged_tables ? " unlogged" : "", p,
-					 partitions, p - 1, ff);
+			printfPQExpBuffer(&query,
+							  "create%s table pgbench_accounts_%d\n"
+							  "  partition of pgbench_accounts\n"
+							  "  for values with (modulus %d, remainder %d)",
+							  unlogged_tables ? " unlogged" : "", p,
+							  partitions, p - 1);
 		else					/* cannot get there */
 			Assert(0);
 
-		executeStatement(con, query);
+		/*
+		 * Per ddlinfo in initCreateTables, fillfactor is needed on table
+		 * pgbench_accounts.
+		 */
+		append_fillfactor(&query);
+
+		executeStatement(con, query.data);
 	}
+
+	termPQExpBuffer(&query);
 }
 
 /*
@@ -3712,48 +3718,39 @@ initCreateTables(PGconn *con)
 			1
 		}
 	};
-	int			i;
+
+	PQExpBufferData		query;
 
 	fprintf(stderr, "creating tables...\n");
 
-	for (i = 0; i < lengthof(DDLs); i++)
+	initPQExpBuffer(&query);
+
+	for (int i = 0; i < lengthof(DDLs); i++)
 	{
-		char		opts[256];
-		char		buffer[256];
 		const struct ddlinfo *ddl = &DDLs[i];
-		const char *cols;
 
 		/* Construct new create table statement. */
-		opts[0] = '\0';
+		printfPQExpBuffer(&query, "create%s table %s(%s)",
+						  unlogged_tables ? " unlogged" : "",
+						  ddl->table,
+						  (scale >= SCALE_32BIT_THRESHOLD) ? ddl->bigcols : ddl->smcols);
 
 		/* Partition pgbench_accounts table */
 		if (partition_method != PART_NONE && strcmp(ddl->table, "pgbench_accounts") == 0)
-			snprintf(opts + strlen(opts), sizeof(opts) - strlen(opts),
-					 " partition by %s (aid)", PARTITION_METHOD[partition_method]);
+			appendPQExpBuffer(&query,
+							  " partition by %s (aid)", PARTITION_METHOD[partition_method]);
 		else if (ddl->declare_fillfactor)
 			/* fillfactor is only expected on actual tables */
-			append_fillfactor(opts, sizeof(opts));
+			append_fillfactor(&query);
 
 		if (tablespace != NULL)
-		{
-			char	   *escape_tablespace;
+			append_tablespace(con, &query, "", tablespace);
 
-			escape_tablespace = PQescapeIdentifier(con, tablespace,
-												   strlen(tablespace));
-			snprintf(opts + strlen(opts), sizeof(opts) - strlen(opts),
-					 " tablespace %s", escape_tablespace);
-			PQfreemem(escape_tablespace);
-		}
-
-		cols = (scale >= SCALE_32BIT_THRESHOLD) ? ddl->bigcols : ddl->smcols;
-
-		snprintf(buffer, sizeof(buffer), "create%s table %s(%s)%s",
-				 unlogged_tables ? " unlogged" : "",
-				 ddl->table, cols, opts);
-
-		executeStatement(con, buffer);
+		executeStatement(con, query.data);
 	}
 
+	termPQExpBuffer(&query);
+
 	if (partition_method != PART_NONE)
 		createPartitions(con);
 }
@@ -3764,10 +3761,23 @@ initCreateTables(PGconn *con)
  * XXX - As default is 100, it could be removed in this case.
  */
 static void
-append_fillfactor(char *opts, int len)
+append_fillfactor(PQExpBuffer query)
 {
-	snprintf(opts + strlen(opts), len - strlen(opts),
-			 " with (fillfactor=%d)", fillfactor);
+	appendPQExpBuffer(query, " with (fillfactor=%d)", fillfactor);
+}
+
+/*
+ * add tablespace option.
+ */
+static void
+append_tablespace(PGconn *con, PQExpBuffer query, const char *kind,
+				  const char *tablespace)
+{
+	char	   *escape_tablespace;
+
+	escape_tablespace = PQescapeIdentifier(con, tablespace, strlen(tablespace));
+	appendPQExpBuffer(query, "%s tablespace %s", kind, escape_tablespace);
+	PQfreemem(escape_tablespace);
 }
 
 /*
@@ -3789,10 +3799,7 @@ initTruncateTables(PGconn *con)
 static void
 initGenerateDataClientSide(PGconn *con)
 {
-	char		sql[256];
-	PGresult   *res;
-	int			i;
-	int64		k;
+	PQExpBufferData	sql;
 
 	/* used to track elapsed time and estimate of the remaining time */
 	instr_time	start,
@@ -3815,50 +3822,47 @@ initGenerateDataClientSide(PGconn *con)
 	/* truncate away any old data */
 	initTruncateTables(con);
 
+	initPQExpBuffer(&sql);
+
 	/*
 	 * fill branches, tellers, accounts in that order in case foreign keys
 	 * already exist
 	 */
-	for (i = 0; i < nbranches * scale; i++)
+	for (int i = 0; i < nbranches * scale; i++)
 	{
 		/* "filler" column defaults to NULL */
-		snprintf(sql, sizeof(sql),
-				 "insert into pgbench_branches(bid,bbalance) values(%d,0)",
-				 i + 1);
-		executeStatement(con, sql);
+		printfPQExpBuffer(&sql,
+						  "insert into pgbench_branches(bid,bbalance) values(%d,0)",
+						  i + 1);
+		executeStatement(con, sql.data);
 	}
 
-	for (i = 0; i < ntellers * scale; i++)
+	for (int i = 0; i < ntellers * scale; i++)
 	{
 		/* "filler" column defaults to NULL */
-		snprintf(sql, sizeof(sql),
-				 "insert into pgbench_tellers(tid,bid,tbalance) values (%d,%d,0)",
-				 i + 1, i / ntellers + 1);
-		executeStatement(con, sql);
+		printfPQExpBuffer(&sql,
+						  "insert into pgbench_tellers(tid,bid,tbalance) values (%d,%d,0)",
+						  i + 1, i / ntellers + 1);
+		executeStatement(con, sql.data);
 	}
 
 	/*
 	 * accounts is big enough to be worth using COPY and tracking runtime
 	 */
-	res = PQexec(con, "copy pgbench_accounts from stdin");
-	if (PQresultStatus(res) != PGRES_COPY_IN)
-	{
-		pg_log_fatal("unexpected copy in result: %s", PQerrorMessage(con));
-		exit(1);
-	}
-	PQclear(res);
+	executeStatementExpect(con, "copy pgbench_accounts from stdin", PGRES_COPY_IN, false);
 
 	INSTR_TIME_SET_CURRENT(start);
 
-	for (k = 0; k < (int64) naccounts * scale; k++)
+	/* printf overheads should probably be avoided... */
+	for (int64 k = 0; k < (int64) naccounts * scale; k++)
 	{
 		int64		j = k + 1;
 
 		/* "filler" column defaults to blank padded empty string */
-		snprintf(sql, sizeof(sql),
-				 INT64_FORMAT "\t" INT64_FORMAT "\t%d\t\n",
-				 j, k / naccounts + 1, 0);
-		if (PQputline(con, sql))
+		printfPQExpBuffer(&sql,
+						  INT64_FORMAT "\t" INT64_FORMAT "\t%d\t\n",
+						  j, k / naccounts + 1, 0);
+		if (PQputline(con, sql.data))
 		{
 			pg_log_fatal("PQputline failed");
 			exit(1);
@@ -3920,6 +3924,8 @@ initGenerateDataClientSide(PGconn *con)
 		exit(1);
 	}
 
+	termPQExpBuffer(&sql);
+
 	executeStatement(con, "commit");
 }
 
@@ -3933,7 +3939,7 @@ initGenerateDataClientSide(PGconn *con)
 static void
 initGenerateDataServerSide(PGconn *con)
 {
-	char		sql[256];
+	PQExpBufferData	sql;
 
 	fprintf(stderr, "generating data (server-side)...\n");
 
@@ -3946,24 +3952,28 @@ initGenerateDataServerSide(PGconn *con)
 	/* truncate away any old data */
 	initTruncateTables(con);
 
-	snprintf(sql, sizeof(sql),
-			 "insert into pgbench_branches(bid,bbalance) "
-			 "select bid, 0 "
-			 "from generate_series(1, %d) as bid", nbranches * scale);
-	executeStatement(con, sql);
+	initPQExpBuffer(&sql);
 
-	snprintf(sql, sizeof(sql),
-			 "insert into pgbench_tellers(tid,bid,tbalance) "
-			 "select tid, (tid - 1) / %d + 1, 0 "
-			 "from generate_series(1, %d) as tid", ntellers, ntellers * scale);
-	executeStatement(con, sql);
+	printfPQExpBuffer(&sql,
+					  "insert into pgbench_branches(bid,bbalance) "
+					  "select bid, 0 "
+					  "from generate_series(1, %d) as bid", nbranches * scale);
+	executeStatement(con, sql.data);
 
-	snprintf(sql, sizeof(sql),
-			 "insert into pgbench_accounts(aid,bid,abalance,filler) "
-			 "select aid, (aid - 1) / %d + 1, 0, '' "
-			 "from generate_series(1, "INT64_FORMAT") as aid",
-			 naccounts, (int64) naccounts * scale);
-	executeStatement(con, sql);
+	printfPQExpBuffer(&sql,
+					  "insert into pgbench_tellers(tid,bid,tbalance) "
+					  "select tid, (tid - 1) / %d + 1, 0 "
+					  "from generate_series(1, %d) as tid", ntellers, ntellers * scale);
+	executeStatement(con, sql.data);
+
+	printfPQExpBuffer(&sql,
+					  "insert into pgbench_accounts(aid,bid,abalance,filler) "
+					  "select aid, (aid - 1) / %d + 1, 0, '' "
+					  "from generate_series(1, "INT64_FORMAT") as aid",
+					  naccounts, (int64) naccounts * scale);
+	executeStatement(con, sql.data);
+
+	termPQExpBuffer(&sql);
 
 	executeStatement(con, "commit");
 }
@@ -3992,28 +4002,23 @@ initCreatePKeys(PGconn *con)
 		"alter table pgbench_tellers add primary key (tid)",
 		"alter table pgbench_accounts add primary key (aid)"
 	};
-	int			i;
+	PQExpBufferData query;
 
 	fprintf(stderr, "creating primary keys...\n");
-	for (i = 0; i < lengthof(DDLINDEXes); i++)
-	{
-		char		buffer[256];
+	initPQExpBuffer(&query);
 
-		strlcpy(buffer, DDLINDEXes[i], sizeof(buffer));
+	for (int i = 0; i < lengthof(DDLINDEXes); i++)
+	{
+		resetPQExpBuffer(&query);
+		appendPQExpBufferStr(&query, DDLINDEXes[i]);
 
 		if (index_tablespace != NULL)
-		{
-			char	   *escape_tablespace;
+			append_tablespace(con, &query, " using index", index_tablespace);
 
-			escape_tablespace = PQescapeIdentifier(con, index_tablespace,
-												   strlen(index_tablespace));
-			snprintf(buffer + strlen(buffer), sizeof(buffer) - strlen(buffer),
-					 " using index tablespace %s", escape_tablespace);
-			PQfreemem(escape_tablespace);
-		}
-
-		executeStatement(con, buffer);
+		executeStatement(con, query.data);
 	}
+
+	termPQExpBuffer(&query);
 }
 
 /*
#14David Steele
david@pgmasters.net
In reply to: Fabien COELHO (#12)
Re: pgbench - refactor init functions with buffers

On 11/6/19 12:48 AM, Fabien COELHO wrote:

Hello Andres,

Attached v3 shorten some lines and adds "append_tablespace".

A v4 which just extends the patch to newly added 'G'.

I'd prefer not to expand the use of pqexpbuffer in more places, and
instead rather see this use StringInfo, now that's also available to
frontend programs.

Franckly, one or the other does not matter much to me.

FWIW, I agree with Andres with regard to using StringInfo.

Also, the changes to executeStatementExpect() and adding
executeStatement() do not seem to fit in with the purpose of this patch.

Regards,
--
-David
david@pgmasters.net

#15Fabien COELHO
coelho@cri.ensmp.fr
In reply to: David Steele (#14)
2 attachment(s)
Re: pgbench - refactor init functions with buffers

Hello David,

I'd prefer not to expand the use of pqexpbuffer in more places, and
instead rather see this use StringInfo, now that's also available to
frontend programs.

Franckly, one or the other does not matter much to me.

FWIW, I agree with Andres with regard to using StringInfo.

Ok. I find it strange to mix PQExpBuffer & StringInfo in the same file.

Also, the changes to executeStatementExpect() and adding executeStatement()
do not seem to fit in with the purpose of this patch.

Yep, that was in passing.

Attached a v6 which uses StringInfo, and the small refactoring as a
separate patch.

--
Fabien.

Attachments:

pgbench-buffer-6.patchtext/x-diff; name=pgbench-buffer-6.patchDownload
diff --git a/src/bin/pgbench/pgbench.c b/src/bin/pgbench/pgbench.c
index b8864c6ae5..40cae7b121 100644
--- a/src/bin/pgbench/pgbench.c
+++ b/src/bin/pgbench/pgbench.c
@@ -62,6 +62,7 @@
 #include "fe_utils/cancel.h"
 #include "fe_utils/conditional.h"
 #include "getopt_long.h"
+#include "lib/stringinfo.h"
 #include "libpq-fe.h"
 #include "pgbench.h"
 #include "portability/instr_time.h"
@@ -599,7 +600,9 @@ static void doLog(TState *thread, CState *st,
 				  StatsData *agg, bool skipped, double latency, double lag);
 static void processXactStats(TState *thread, CState *st, instr_time *now,
 							 bool skipped, StatsData *agg);
-static void append_fillfactor(char *opts, int len);
+static void append_fillfactor(StringInfo query);
+static void append_tablespace(PGconn *con, StringInfo query,
+							  const char *kind, const char *tablespace);
 static void addScript(ParsedScript script);
 static void *threadRun(void *arg);
 static void finishCon(CState *st);
@@ -3608,30 +3611,27 @@ initDropTables(PGconn *con)
 static void
 createPartitions(PGconn *con)
 {
-	char		ff[64];
-
-	ff[0] = '\0';
-
-	/*
-	 * Per ddlinfo in initCreateTables, fillfactor is needed on table
-	 * pgbench_accounts.
-	 */
-	append_fillfactor(ff, sizeof(ff));
+	StringInfoData		query;
 
 	/* we must have to create some partitions */
 	Assert(partitions > 0);
 
 	fprintf(stderr, "creating %d partitions...\n", partitions);
 
+	initStringInfo(&query);
+
 	for (int p = 1; p <= partitions; p++)
 	{
-		char		query[256];
-
 		if (partition_method == PART_RANGE)
 		{
 			int64		part_size = (naccounts * (int64) scale + partitions - 1) / partitions;
-			char		minvalue[32],
-						maxvalue[32];
+
+			resetStringInfo(&query);
+			appendStringInfo(&query,
+							 "create%s table pgbench_accounts_%d\n"
+							 "  partition of pgbench_accounts\n"
+							 "  for values from (",
+							 unlogged_tables ? " unlogged" : "", p);
 
 			/*
 			 * For RANGE, we use open-ended partitions at the beginning and
@@ -3640,34 +3640,42 @@ createPartitions(PGconn *con)
 			 * scale, it is more generic and the performance is better.
 			 */
 			if (p == 1)
-				sprintf(minvalue, "minvalue");
+				appendStringInfoString(&query, "minvalue");
 			else
-				sprintf(minvalue, INT64_FORMAT, (p - 1) * part_size + 1);
+				appendStringInfo(&query, INT64_FORMAT, (p - 1) * part_size + 1);
+
+			appendStringInfoString(&query, ") to (");
 
 			if (p < partitions)
-				sprintf(maxvalue, INT64_FORMAT, p * part_size + 1);
+				appendStringInfo(&query, INT64_FORMAT, p * part_size + 1);
 			else
-				sprintf(maxvalue, "maxvalue");
+				appendStringInfoString(&query, "maxvalue");
 
-			snprintf(query, sizeof(query),
-					 "create%s table pgbench_accounts_%d\n"
-					 "  partition of pgbench_accounts\n"
-					 "  for values from (%s) to (%s)%s\n",
-					 unlogged_tables ? " unlogged" : "", p,
-					 minvalue, maxvalue, ff);
+			appendStringInfoChar(&query, ')');
 		}
 		else if (partition_method == PART_HASH)
-			snprintf(query, sizeof(query),
-					 "create%s table pgbench_accounts_%d\n"
-					 "  partition of pgbench_accounts\n"
-					 "  for values with (modulus %d, remainder %d)%s\n",
-					 unlogged_tables ? " unlogged" : "", p,
-					 partitions, p - 1, ff);
+		{
+			resetStringInfo(&query);
+			appendStringInfo(&query,
+							 "create%s table pgbench_accounts_%d\n"
+							 "  partition of pgbench_accounts\n"
+							 "  for values with (modulus %d, remainder %d)",
+							 unlogged_tables ? " unlogged" : "", p,
+							 partitions, p - 1);
+		}
 		else					/* cannot get there */
 			Assert(0);
 
-		executeStatement(con, query);
+		/*
+		 * Per ddlinfo in initCreateTables, fillfactor is needed on table
+		 * pgbench_accounts.
+		 */
+		append_fillfactor(&query);
+
+		executeStatement(con, query.data);
 	}
+
+	pfree(query.data);
 }
 
 /*
@@ -3720,48 +3728,40 @@ initCreateTables(PGconn *con)
 			1
 		}
 	};
-	int			i;
+
+	StringInfoData		query;
 
 	fprintf(stderr, "creating tables...\n");
 
-	for (i = 0; i < lengthof(DDLs); i++)
+	initStringInfo(&query);
+
+	for (int i = 0; i < lengthof(DDLs); i++)
 	{
-		char		opts[256];
-		char		buffer[256];
 		const struct ddlinfo *ddl = &DDLs[i];
-		const char *cols;
 
 		/* Construct new create table statement. */
-		opts[0] = '\0';
+		resetStringInfo(&query);
+		appendStringInfo(&query, "create%s table %s(%s)",
+						 unlogged_tables ? " unlogged" : "",
+						 ddl->table,
+						 (scale >= SCALE_32BIT_THRESHOLD) ? ddl->bigcols : ddl->smcols);
 
 		/* Partition pgbench_accounts table */
 		if (partition_method != PART_NONE && strcmp(ddl->table, "pgbench_accounts") == 0)
-			snprintf(opts + strlen(opts), sizeof(opts) - strlen(opts),
-					 " partition by %s (aid)", PARTITION_METHOD[partition_method]);
+			appendStringInfo(&query,
+							 " partition by %s (aid)", PARTITION_METHOD[partition_method]);
 		else if (ddl->declare_fillfactor)
 			/* fillfactor is only expected on actual tables */
-			append_fillfactor(opts, sizeof(opts));
+			append_fillfactor(&query);
 
 		if (tablespace != NULL)
-		{
-			char	   *escape_tablespace;
+			append_tablespace(con, &query, "", tablespace);
 
-			escape_tablespace = PQescapeIdentifier(con, tablespace,
-												   strlen(tablespace));
-			snprintf(opts + strlen(opts), sizeof(opts) - strlen(opts),
-					 " tablespace %s", escape_tablespace);
-			PQfreemem(escape_tablespace);
-		}
-
-		cols = (scale >= SCALE_32BIT_THRESHOLD) ? ddl->bigcols : ddl->smcols;
-
-		snprintf(buffer, sizeof(buffer), "create%s table %s(%s)%s",
-				 unlogged_tables ? " unlogged" : "",
-				 ddl->table, cols, opts);
-
-		executeStatement(con, buffer);
+		executeStatement(con, query.data);
 	}
 
+	pfree(query.data);
+
 	if (partition_method != PART_NONE)
 		createPartitions(con);
 }
@@ -3772,10 +3772,23 @@ initCreateTables(PGconn *con)
  * XXX - As default is 100, it could be removed in this case.
  */
 static void
-append_fillfactor(char *opts, int len)
+append_fillfactor(StringInfo query)
 {
-	snprintf(opts + strlen(opts), len - strlen(opts),
-			 " with (fillfactor=%d)", fillfactor);
+	appendStringInfo(query, " with (fillfactor=%d)", fillfactor);
+}
+
+/*
+ * add tablespace option.
+ */
+static void
+append_tablespace(PGconn *con, StringInfo query, const char *kind,
+				  const char *tablespace)
+{
+	char	   *escape_tablespace;
+
+	escape_tablespace = PQescapeIdentifier(con, tablespace, strlen(tablespace));
+	appendStringInfo(query, "%s tablespace %s", kind, escape_tablespace);
+	PQfreemem(escape_tablespace);
 }
 
 /*
@@ -3797,10 +3810,8 @@ initTruncateTables(PGconn *con)
 static void
 initGenerateDataClientSide(PGconn *con)
 {
-	char		sql[256];
-	PGresult   *res;
-	int			i;
-	int64		k;
+	PGresult	   *res;
+	StringInfoData	sql;
 
 	/* used to track elapsed time and estimate of the remaining time */
 	instr_time	start,
@@ -3823,26 +3834,30 @@ initGenerateDataClientSide(PGconn *con)
 	/* truncate away any old data */
 	initTruncateTables(con);
 
+	initStringInfo(&sql);
+
 	/*
 	 * fill branches, tellers, accounts in that order in case foreign keys
 	 * already exist
 	 */
-	for (i = 0; i < nbranches * scale; i++)
+	for (int i = 0; i < nbranches * scale; i++)
 	{
 		/* "filler" column defaults to NULL */
-		snprintf(sql, sizeof(sql),
-				 "insert into pgbench_branches(bid,bbalance) values(%d,0)",
-				 i + 1);
-		executeStatement(con, sql);
+		resetStringInfo(&sql);
+		appendStringInfo(&sql,
+						 "insert into pgbench_branches(bid,bbalance) values(%d,0)",
+						 i + 1);
+		executeStatement(con, sql.data);
 	}
 
-	for (i = 0; i < ntellers * scale; i++)
+	for (int i = 0; i < ntellers * scale; i++)
 	{
 		/* "filler" column defaults to NULL */
-		snprintf(sql, sizeof(sql),
-				 "insert into pgbench_tellers(tid,bid,tbalance) values (%d,%d,0)",
-				 i + 1, i / ntellers + 1);
-		executeStatement(con, sql);
+		resetStringInfo(&sql);
+		appendStringInfo(&sql,
+						 "insert into pgbench_tellers(tid,bid,tbalance) values (%d,%d,0)",
+						 i + 1, i / ntellers + 1);
+		executeStatement(con, sql.data);
 	}
 
 	/*
@@ -3858,15 +3873,17 @@ initGenerateDataClientSide(PGconn *con)
 
 	INSTR_TIME_SET_CURRENT(start);
 
-	for (k = 0; k < (int64) naccounts * scale; k++)
+	/* printf overheads should probably be avoided... */
+	for (int64 k = 0; k < (int64) naccounts * scale; k++)
 	{
 		int64		j = k + 1;
 
 		/* "filler" column defaults to blank padded empty string */
-		snprintf(sql, sizeof(sql),
-				 INT64_FORMAT "\t" INT64_FORMAT "\t%d\t\n",
-				 j, k / naccounts + 1, 0);
-		if (PQputline(con, sql))
+		resetStringInfo(&sql);
+		appendStringInfo(&sql,
+						 INT64_FORMAT "\t" INT64_FORMAT "\t%d\t\n",
+						 j, k / naccounts + 1, 0);
+		if (PQputline(con, sql.data))
 		{
 			pg_log_fatal("PQputline failed");
 			exit(1);
@@ -3928,6 +3945,8 @@ initGenerateDataClientSide(PGconn *con)
 		exit(1);
 	}
 
+	pfree(sql.data);
+
 	executeStatement(con, "commit");
 }
 
@@ -3941,7 +3960,7 @@ initGenerateDataClientSide(PGconn *con)
 static void
 initGenerateDataServerSide(PGconn *con)
 {
-	char		sql[256];
+	StringInfoData	sql;
 
 	fprintf(stderr, "generating data (server-side)...\n");
 
@@ -3954,24 +3973,30 @@ initGenerateDataServerSide(PGconn *con)
 	/* truncate away any old data */
 	initTruncateTables(con);
 
-	snprintf(sql, sizeof(sql),
-			 "insert into pgbench_branches(bid,bbalance) "
-			 "select bid, 0 "
-			 "from generate_series(1, %d) as bid", nbranches * scale);
-	executeStatement(con, sql);
+	initStringInfo(&sql);
 
-	snprintf(sql, sizeof(sql),
-			 "insert into pgbench_tellers(tid,bid,tbalance) "
-			 "select tid, (tid - 1) / %d + 1, 0 "
-			 "from generate_series(1, %d) as tid", ntellers, ntellers * scale);
-	executeStatement(con, sql);
+	appendStringInfo(&sql,
+					 "insert into pgbench_branches(bid,bbalance) "
+					 "select bid, 0 "
+					 "from generate_series(1, %d) as bid", nbranches * scale);
+	executeStatement(con, sql.data);
 
-	snprintf(sql, sizeof(sql),
-			 "insert into pgbench_accounts(aid,bid,abalance,filler) "
-			 "select aid, (aid - 1) / %d + 1, 0, '' "
-			 "from generate_series(1, "INT64_FORMAT") as aid",
-			 naccounts, (int64) naccounts * scale);
-	executeStatement(con, sql);
+	resetStringInfo(&sql);
+	appendStringInfo(&sql,
+					 "insert into pgbench_tellers(tid,bid,tbalance) "
+					 "select tid, (tid - 1) / %d + 1, 0 "
+					 "from generate_series(1, %d) as tid", ntellers, ntellers * scale);
+	executeStatement(con, sql.data);
+
+	resetStringInfo(&sql);
+	appendStringInfo(&sql,
+					 "insert into pgbench_accounts(aid,bid,abalance,filler) "
+					 "select aid, (aid - 1) / %d + 1, 0, '' "
+					 "from generate_series(1, "INT64_FORMAT") as aid",
+					 naccounts, (int64) naccounts * scale);
+	executeStatement(con, sql.data);
+
+	pfree(sql.data);
 
 	executeStatement(con, "commit");
 }
@@ -4000,28 +4025,23 @@ initCreatePKeys(PGconn *con)
 		"alter table pgbench_tellers add primary key (tid)",
 		"alter table pgbench_accounts add primary key (aid)"
 	};
-	int			i;
+	StringInfoData query;
 
 	fprintf(stderr, "creating primary keys...\n");
-	for (i = 0; i < lengthof(DDLINDEXes); i++)
-	{
-		char		buffer[256];
+	initStringInfo(&query);
 
-		strlcpy(buffer, DDLINDEXes[i], sizeof(buffer));
+	for (int i = 0; i < lengthof(DDLINDEXes); i++)
+	{
+		resetStringInfo(&query);
+		appendStringInfoString(&query, DDLINDEXes[i]);
 
 		if (index_tablespace != NULL)
-		{
-			char	   *escape_tablespace;
+			append_tablespace(con, &query, " using index", index_tablespace);
 
-			escape_tablespace = PQescapeIdentifier(con, index_tablespace,
-												   strlen(index_tablespace));
-			snprintf(buffer + strlen(buffer), sizeof(buffer) - strlen(buffer),
-					 " using index tablespace %s", escape_tablespace);
-			PQfreemem(escape_tablespace);
-		}
-
-		executeStatement(con, buffer);
+		executeStatement(con, query.data);
 	}
+
+	pfree(query.data);
 }
 
 /*
@@ -4079,13 +4099,13 @@ checkInitSteps(const char *initialize_steps)
 static void
 runInitSteps(const char *initialize_steps)
 {
-	PQExpBufferData stats;
+	StringInfoData stats;
 	PGconn	   *con;
 	const char *step;
 	double		run_time = 0.0;
 	bool		first = true;
 
-	initPQExpBuffer(&stats);
+	initStringInfo(&stats);
 
 	if ((con = doConnect()) == NULL)
 		exit(1);
@@ -4148,11 +4168,11 @@ runInitSteps(const char *initialize_steps)
 			elapsed_sec = INSTR_TIME_GET_DOUBLE(diff);
 
 			if (!first)
-				appendPQExpBufferStr(&stats, ", ");
+				appendStringInfoString(&stats, ", ");
 			else
 				first = false;
 
-			appendPQExpBuffer(&stats, "%s %.2f s", op, elapsed_sec);
+			appendStringInfo(&stats, "%s %.2f s", op, elapsed_sec);
 
 			run_time += elapsed_sec;
 		}
@@ -4161,7 +4181,7 @@ runInitSteps(const char *initialize_steps)
 	fprintf(stderr, "done in %.2f s (%s).\n", run_time, stats.data);
 	ResetCancelConn();
 	PQfinish(con);
-	termPQExpBuffer(&stats);
+	pfree(stats.data);
 }
 
 /*
pgbench-refactor-exec-1.patchtext/x-diff; name=pgbench-refactor-exec-1.patchDownload
diff --git a/src/bin/pgbench/pgbench.c b/src/bin/pgbench/pgbench.c
index 40cae7b121..b773f14f7e 100644
--- a/src/bin/pgbench/pgbench.c
+++ b/src/bin/pgbench/pgbench.c
@@ -1137,35 +1137,38 @@ accumStats(StatsData *stats, bool skipped, double lat, double lag)
 	}
 }
 
-/* call PQexec() and exit() on failure */
+/* call PQexec() and possibly exit() on failure */
 static void
-executeStatement(PGconn *con, const char *sql)
+executeStatementExpect(PGconn *con, const char *sql,
+					   const ExecStatusType expected, bool errorOK)
 {
 	PGresult   *res;
 
 	res = PQexec(con, sql);
-	if (PQresultStatus(res) != PGRES_COMMAND_OK)
+	if (PQresultStatus(res) != expected)
 	{
 		pg_log_fatal("query failed: %s", PQerrorMessage(con));
 		pg_log_info("query was: %s", sql);
-		exit(1);
+		if (errorOK)
+			pg_log_info("(ignoring this error and continuing anyway)");
+		else
+			exit(1);
 	}
 	PQclear(res);
 }
 
+/* execute sql command, which must work, or die if not */
+static void
+executeStatement(PGconn *con, const char *sql)
+{
+	executeStatementExpect(con, sql, PGRES_COMMAND_OK, false);
+}
+
 /* call PQexec() and complain, but without exiting, on failure */
 static void
 tryExecuteStatement(PGconn *con, const char *sql)
 {
-	PGresult   *res;
-
-	res = PQexec(con, sql);
-	if (PQresultStatus(res) != PGRES_COMMAND_OK)
-	{
-		pg_log_error("%s", PQerrorMessage(con));
-		pg_log_info("(ignoring this error and continuing anyway)");
-	}
-	PQclear(res);
+	executeStatementExpect(con, sql, PGRES_COMMAND_OK, true);
 }
 
 /* set up a connection to the backend */
@@ -3810,7 +3813,6 @@ initTruncateTables(PGconn *con)
 static void
 initGenerateDataClientSide(PGconn *con)
 {
-	PGresult	   *res;
 	StringInfoData	sql;
 
 	/* used to track elapsed time and estimate of the remaining time */
@@ -3863,13 +3865,7 @@ initGenerateDataClientSide(PGconn *con)
 	/*
 	 * accounts is big enough to be worth using COPY and tracking runtime
 	 */
-	res = PQexec(con, "copy pgbench_accounts from stdin");
-	if (PQresultStatus(res) != PGRES_COPY_IN)
-	{
-		pg_log_fatal("unexpected copy in result: %s", PQerrorMessage(con));
-		exit(1);
-	}
-	PQclear(res);
+	executeStatementExpect(con, "copy pgbench_accounts from stdin", PGRES_COPY_IN, false);
 
 	INSTR_TIME_SET_CURRENT(start);
 
#16David Steele
david@pgmasters.net
In reply to: Fabien COELHO (#15)
Re: pgbench - refactor init functions with buffers

On 3/27/20 6:13 PM, Fabien COELHO wrote:

Hello David,

I'd prefer not to expand the use of pqexpbuffer in more places, and
instead rather see this use StringInfo, now that's also available to
frontend programs.

Franckly, one or the other does not matter much to me.

FWIW, I agree with Andres with regard to using StringInfo.

Ok. I find it strange to mix PQExpBuffer & StringInfo in the same file.

Agreed, but we'd rather use StringInfo going forward. However, I don't
think that puts you on the hook for updating all the PQExpBuffer references.

Unless you want to...

Also, the changes to executeStatementExpect() and adding
executeStatement() do not seem to fit in with the purpose of this patch.

Yep, that was in passing.

Attached a v6 which uses StringInfo, and the small refactoring as a
separate patch.

I think that's better, thanks.

Regards,
--
-David
david@pgmasters.net

#17Fabien COELHO
coelho@cri.ensmp.fr
In reply to: David Steele (#16)
Re: pgbench - refactor init functions with buffers

Ok. I find it strange to mix PQExpBuffer & StringInfo in the same file.

Agreed, but we'd rather use StringInfo going forward. However, I don't think
that puts you on the hook for updating all the PQExpBuffer references.

Unless you want to...

I cannot say that I "want" to fix something which already works the same
way, because it is against my coding principles.

However there may be some fun in writing a little script to replace one
with the other automatically. I counted nearly 3500 calls under src/bin.

--
Fabien.

#18Tom Lane
tgl@sss.pgh.pa.us
In reply to: Fabien COELHO (#17)
Re: pgbench - refactor init functions with buffers

Fabien COELHO <coelho@cri.ensmp.fr> writes:

Ok. I find it strange to mix PQExpBuffer & StringInfo in the same file.

Agreed, but we'd rather use StringInfo going forward. However, I don't think
that puts you on the hook for updating all the PQExpBuffer references.
Unless you want to...

I cannot say that I "want" to fix something which already works the same
way, because it is against my coding principles.
However there may be some fun in writing a little script to replace one
with the other automatically. I counted nearly 3500 calls under src/bin.

Yeah, that's the problem. If someone does come forward with a patch to do
that, I think it'd be summarily rejected, at least in high-traffic code
like pg_dump. The pain it'd cause for back-patching would outweigh the
value.

That being the case, I'd think a better design principle is "make your
new code look like the code around it", which would tend to weigh against
introducing StringInfo uses into pgbench when there's none there now and
a bunch of PQExpBuffer instead. So I can't help thinking the advice
you're being given here is suspect.

regards, tom lane

#19Alvaro Herrera
alvherre@2ndquadrant.com
In reply to: Tom Lane (#18)
Re: pgbench - refactor init functions with buffers

On 2020-Mar-27, Tom Lane wrote:

That being the case, I'd think a better design principle is "make your
new code look like the code around it", which would tend to weigh against
introducing StringInfo uses into pgbench when there's none there now and
a bunch of PQExpBuffer instead. So I can't help thinking the advice
you're being given here is suspect.

+1 for keeping it PQExpBuffer-only, until such a time when you need a
StringInfo feature that's not in PQExpBuffer -- and even at that point,
I think you'd switch just that one thing to StringInfo, not the whole
program.

--
�lvaro Herrera https://www.2ndQuadrant.com/
PostgreSQL Development, 24x7 Support, Remote DBA, Training & Services

#20Fabien COELHO
coelho@cri.ensmp.fr
In reply to: Tom Lane (#18)
Re: pgbench - refactor init functions with buffers

Hello Tom,

I cannot say that I "want" to fix something which already works the same
way, because it is against my coding principles. [...]
I counted nearly 3500 calls under src/bin.

Yeah, that's the problem. If someone does come forward with a patch to do
that, I think it'd be summarily rejected, at least in high-traffic code
like pg_dump. The pain it'd cause for back-patching would outweigh the
value.

What about "typedef StringInfoData PQExpBufferData" and replacing
PQExpBuffer by StringInfo internally, just keeping the old interface
around because it is there? That would remove a few hundreds clocs.

ISTM that with inline and varargs macro the substition can be managed
reasonably lightly, depending on what level of compatibility is required
for libpq: should it be linkability, or requiring a recompilation is ok?

A clear benefit is that there are quite a few utils for PQExpBuffer in
"fe_utils/string_utils.c" which would become available for StringInfo,
which would help using StringInfo without duplicating them.

That being the case, I'd think a better design principle is "make your
new code look like the code around it",

Yep.

which would tend to weigh against introducing StringInfo uses into
pgbench when there's none there now and a bunch of PQExpBuffer instead.
So I can't help thinking the advice you're being given here is suspect.

Well, that is what I was saying, but at 2 against 1, I fold.

--
Fabien.

#21David Steele
david@pgmasters.net
In reply to: Alvaro Herrera (#19)
Re: pgbench - refactor init functions with buffers

On 3/27/20 9:52 PM, Alvaro Herrera wrote:

On 2020-Mar-27, Tom Lane wrote:

That being the case, I'd think a better design principle is "make your
new code look like the code around it", which would tend to weigh against
introducing StringInfo uses into pgbench when there's none there now and
a bunch of PQExpBuffer instead. So I can't help thinking the advice
you're being given here is suspect.

+1 for keeping it PQExpBuffer-only, until such a time when you need a
StringInfo feature that's not in PQExpBuffer -- and even at that point,
I think you'd switch just that one thing to StringInfo, not the whole
program.

I think I need to be careful what I joke about. It wasn't my intention
to advocate changing all the existing *PQExpBuffer() calls in bin.

But, the only prior committer to look at this patch expressed a
preference for StringInfo so in the absence of any other input I thought
it might move the patch forward if I reinforced that. Now it seems the
consensus has moved in favor of *PQExpBuffer().

Fabien has provided a patch in each flavor, so I guess the question is:
is it committable either way?

Regards,
--
-David
david@pgmasters.net

#22Andres Freund
andres@anarazel.de
In reply to: Tom Lane (#18)
Re: pgbench - refactor init functions with buffers

Hi,

On 2020-03-27 19:57:12 -0400, Tom Lane wrote:

Fabien COELHO <coelho@cri.ensmp.fr> writes:

Ok. I find it strange to mix PQExpBuffer & StringInfo in the same file.

Agreed, but we'd rather use StringInfo going forward. However, I don't think
that puts you on the hook for updating all the PQExpBuffer references.
Unless you want to...

I cannot say that I "want" to fix something which already works the same
way, because it is against my coding principles.
However there may be some fun in writing a little script to replace one
with the other automatically. I counted nearly 3500 calls under src/bin.

Yeah, that's the problem. If someone does come forward with a patch to do
that, I think it'd be summarily rejected, at least in high-traffic code
like pg_dump. The pain it'd cause for back-patching would outweigh the
value.

Sure, but that's not at all what was proposed.

That being the case, I'd think a better design principle is "make your
new code look like the code around it", which would tend to weigh against
introducing StringInfo uses into pgbench when there's none there now and
a bunch of PQExpBuffer instead. So I can't help thinking the advice
you're being given here is suspect.

I don't agree with this. This is a "fresh" usage of StringInfo. That's
different to adding one new printed line among others built with
pqexpbuffer. If we continue adding large numbers of new uses of both
pieces of infrastructure, we're just making things more confusing.

Greetings,

Andres Freund

#23Tom Lane
tgl@sss.pgh.pa.us
In reply to: Andres Freund (#22)
Re: pgbench - refactor init functions with buffers

Andres Freund <andres@anarazel.de> writes:

On 2020-03-27 19:57:12 -0400, Tom Lane wrote:

That being the case, I'd think a better design principle is "make your
new code look like the code around it", which would tend to weigh against
introducing StringInfo uses into pgbench when there's none there now and
a bunch of PQExpBuffer instead. So I can't help thinking the advice
you're being given here is suspect.

I don't agree with this. This is a "fresh" usage of StringInfo. That's
different to adding one new printed line among others built with
pqexpbuffer. If we continue adding large numbers of new uses of both
pieces of infrastructure, we're just making things more confusing.

Why? I'm not aware of any intention to deprecate/remove PQExpBuffer,
and I doubt it'd be a good thing to try. It does some things that
StringInfo won't, notably cope with OOM without crashing.

regards, tom lane

#24Andres Freund
andres@anarazel.de
In reply to: Tom Lane (#23)
Re: pgbench - refactor init functions with buffers

On 2020-03-28 14:49:31 -0400, Tom Lane wrote:

Andres Freund <andres@anarazel.de> writes:

On 2020-03-27 19:57:12 -0400, Tom Lane wrote:

That being the case, I'd think a better design principle is "make your
new code look like the code around it", which would tend to weigh against
introducing StringInfo uses into pgbench when there's none there now and
a bunch of PQExpBuffer instead. So I can't help thinking the advice
you're being given here is suspect.

I don't agree with this. This is a "fresh" usage of StringInfo. That's
different to adding one new printed line among others built with
pqexpbuffer. If we continue adding large numbers of new uses of both
pieces of infrastructure, we're just making things more confusing.

Why? I'm not aware of any intention to deprecate/remove PQExpBuffer,
and I doubt it'd be a good thing to try. It does some things that
StringInfo won't, notably cope with OOM without crashing.

- code using it cannot easily be shared between frontend/backend (no
memory context integration etc)
- most code does *not* want to deal with the potential for OOM without
erroring out
- it's naming is even more confusing than StringInfo
- it introduces dependencies to libpq even when not needed
- both stringinfo and pqexpbuffer are performance relevant in some uses,
needing to optimize both is wasted effort
- we shouldn't expose everyone to both APIs except where needed - it's
stuff one has to learn

#25Tom Lane
tgl@sss.pgh.pa.us
In reply to: Andres Freund (#24)
Re: pgbench - refactor init functions with buffers

Andres Freund <andres@anarazel.de> writes:

On 2020-03-28 14:49:31 -0400, Tom Lane wrote:

Why? I'm not aware of any intention to deprecate/remove PQExpBuffer,
and I doubt it'd be a good thing to try. It does some things that
StringInfo won't, notably cope with OOM without crashing.

- code using it cannot easily be shared between frontend/backend (no
memory context integration etc)

True, but also pretty irrelevant for pgbench and similar code.

- most code does *not* want to deal with the potential for OOM without
erroring out

Fair point.

- it's naming is even more confusing than StringInfo

Eye of the beholder ...

- it introduces dependencies to libpq even when not needed

Most of our FE programs do include libpq, and pgbench certainly does,
so this seems like a pretty irrelevant objection as well.

- both stringinfo and pqexpbuffer are performance relevant in some uses,
needing to optimize both is wasted effort

I'm not aware that anybody is trying to micro-optimize either. Even
if someone is, it doesn't imply that they need to change both.

- we shouldn't expose everyone to both APIs except where needed - it's
stuff one has to learn

That situation is unlikely to change in the foreseeable future.
Moreover, using both APIs in one program, where we were not before,
makes it worse not better.

regards, tom lane

#26Andres Freund
andres@anarazel.de
In reply to: Tom Lane (#25)
Re: pgbench - refactor init functions with buffers

On 2020-03-28 15:16:21 -0400, Tom Lane wrote:

Andres Freund <andres@anarazel.de> writes:

- both stringinfo and pqexpbuffer are performance relevant in some uses,
needing to optimize both is wasted effort

I'm not aware that anybody is trying to micro-optimize either.

/messages/by-id/5450.1578797036@sss.pgh.pa.us

#27Fabien COELHO
coelho@cri.ensmp.fr
In reply to: Andres Freund (#22)
Re: pgbench - refactor init functions with buffers

Hello Andres,

That being the case, I'd think a better design principle is "make your
new code look like the code around it", which would tend to weigh against
introducing StringInfo uses into pgbench when there's none there now and
a bunch of PQExpBuffer instead. So I can't help thinking the advice
you're being given here is suspect.

I don't agree with this. This is a "fresh" usage of StringInfo. That's
different to adding one new printed line among others built with
pqexpbuffer. If we continue adding large numbers of new uses of both
pieces of infrastructure, we're just making things more confusing.

My 0.02ᅵᅵ :

- I'm in favor or having one tool for one purpose, so a fe/be common
StringInfo interface is fine with me;

- I prefer to avoid using both PQExpBuffer & StringInfo in the same file,
because they do the exact same thing and it is locally confusing;

- I'd be fine with switching all of pgbench to StringInfo, as there are
only 31 uses;

- But, pgbench relies on psql scanner, which uses PQExpBuffer in
PsqlScanState, so mixing is unavoidable, unless PQExpBuffer & StringInfo
are the same thing (i.e. typedef + cpp/inline/function wrappers);

- There are 1260 uses of PQExpBuffer in psql that, although they are
trivial, I'm in no hurry to update.

--
Fabien.

#28Fabien COELHO
coelho@cri.ensmp.fr
In reply to: David Steele (#21)
1 attachment(s)
Re: pgbench - refactor init functions with buffers

in favor of *PQExpBuffer().

Attached v7 is rebased v5 which uses PQExpBuffer, per cfbot.

--
Fabien.

Attachments:

pgbench-buffer-7.patchtext/x-diff; name=pgbench-buffer-7.patchDownload
diff --git a/src/bin/pgbench/pgbench.c b/src/bin/pgbench/pgbench.c
index 08a5947a9e..3abc41954a 100644
--- a/src/bin/pgbench/pgbench.c
+++ b/src/bin/pgbench/pgbench.c
@@ -602,7 +602,9 @@ static void doLog(TState *thread, CState *st,
 				  StatsData *agg, bool skipped, double latency, double lag);
 static void processXactStats(TState *thread, CState *st, instr_time *now,
 							 bool skipped, StatsData *agg);
-static void append_fillfactor(char *opts, int len);
+static void append_fillfactor(PQExpBuffer query);
+static void append_tablespace(PGconn *con, PQExpBuffer query,
+							  const char *kind, const char *tablespace);
 static void addScript(ParsedScript script);
 static void *threadRun(void *arg);
 static void finishCon(CState *st);
@@ -1137,35 +1139,38 @@ accumStats(StatsData *stats, bool skipped, double lat, double lag)
 	}
 }
 
-/* call PQexec() and exit() on failure */
+/* call PQexec() and possibly exit() on failure */
 static void
-executeStatement(PGconn *con, const char *sql)
+executeStatementExpect(PGconn *con, const char *sql,
+					   const ExecStatusType expected, bool errorOK)
 {
 	PGresult   *res;
 
 	res = PQexec(con, sql);
-	if (PQresultStatus(res) != PGRES_COMMAND_OK)
+	if (PQresultStatus(res) != expected)
 	{
 		pg_log_fatal("query failed: %s", PQerrorMessage(con));
 		pg_log_info("query was: %s", sql);
-		exit(1);
+		if (errorOK)
+			pg_log_info("(ignoring this error and continuing anyway)");
+		else
+			exit(1);
 	}
 	PQclear(res);
 }
 
+/* execute sql command, which must work, or die if not */
+static void
+executeStatement(PGconn *con, const char *sql)
+{
+	executeStatementExpect(con, sql, PGRES_COMMAND_OK, false);
+}
+
 /* call PQexec() and complain, but without exiting, on failure */
 static void
 tryExecuteStatement(PGconn *con, const char *sql)
 {
-	PGresult   *res;
-
-	res = PQexec(con, sql);
-	if (PQresultStatus(res) != PGRES_COMMAND_OK)
-	{
-		pg_log_error("%s", PQerrorMessage(con));
-		pg_log_info("(ignoring this error and continuing anyway)");
-	}
-	PQclear(res);
+	executeStatementExpect(con, sql, PGRES_COMMAND_OK, true);
 }
 
 /* set up a connection to the backend */
@@ -3631,30 +3636,26 @@ initDropTables(PGconn *con)
 static void
 createPartitions(PGconn *con)
 {
-	char		ff[64];
-
-	ff[0] = '\0';
-
-	/*
-	 * Per ddlinfo in initCreateTables, fillfactor is needed on table
-	 * pgbench_accounts.
-	 */
-	append_fillfactor(ff, sizeof(ff));
+	PQExpBufferData		query;
 
 	/* we must have to create some partitions */
 	Assert(partitions > 0);
 
 	fprintf(stderr, "creating %d partitions...\n", partitions);
 
+	initPQExpBuffer(&query);
+
 	for (int p = 1; p <= partitions; p++)
 	{
-		char		query[256];
-
 		if (partition_method == PART_RANGE)
 		{
 			int64		part_size = (naccounts * (int64) scale + partitions - 1) / partitions;
-			char		minvalue[32],
-						maxvalue[32];
+
+			printfPQExpBuffer(&query,
+							  "create%s table pgbench_accounts_%d\n"
+							  "  partition of pgbench_accounts\n"
+							  "  for values from (",
+							  unlogged_tables ? " unlogged" : "", p);
 
 			/*
 			 * For RANGE, we use open-ended partitions at the beginning and
@@ -3663,34 +3664,39 @@ createPartitions(PGconn *con)
 			 * scale, it is more generic and the performance is better.
 			 */
 			if (p == 1)
-				sprintf(minvalue, "minvalue");
+				appendPQExpBufferStr(&query, "minvalue");
 			else
-				sprintf(minvalue, INT64_FORMAT, (p - 1) * part_size + 1);
+				appendPQExpBuffer(&query, INT64_FORMAT, (p - 1) * part_size + 1);
+
+			appendPQExpBufferStr(&query, ") to (");
 
 			if (p < partitions)
-				sprintf(maxvalue, INT64_FORMAT, p * part_size + 1);
+				appendPQExpBuffer(&query, INT64_FORMAT, p * part_size + 1);
 			else
-				sprintf(maxvalue, "maxvalue");
+				appendPQExpBufferStr(&query, "maxvalue");
 
-			snprintf(query, sizeof(query),
-					 "create%s table pgbench_accounts_%d\n"
-					 "  partition of pgbench_accounts\n"
-					 "  for values from (%s) to (%s)%s\n",
-					 unlogged_tables ? " unlogged" : "", p,
-					 minvalue, maxvalue, ff);
+			appendPQExpBufferChar(&query, ')');
 		}
 		else if (partition_method == PART_HASH)
-			snprintf(query, sizeof(query),
-					 "create%s table pgbench_accounts_%d\n"
-					 "  partition of pgbench_accounts\n"
-					 "  for values with (modulus %d, remainder %d)%s\n",
-					 unlogged_tables ? " unlogged" : "", p,
-					 partitions, p - 1, ff);
+			printfPQExpBuffer(&query,
+							  "create%s table pgbench_accounts_%d\n"
+							  "  partition of pgbench_accounts\n"
+							  "  for values with (modulus %d, remainder %d)",
+							  unlogged_tables ? " unlogged" : "", p,
+							  partitions, p - 1);
 		else					/* cannot get there */
 			Assert(0);
 
-		executeStatement(con, query);
+		/*
+		 * Per ddlinfo in initCreateTables, fillfactor is needed on table
+		 * pgbench_accounts.
+		 */
+		append_fillfactor(&query);
+
+		executeStatement(con, query.data);
 	}
+
+	termPQExpBuffer(&query);
 }
 
 /*
@@ -3743,48 +3749,39 @@ initCreateTables(PGconn *con)
 			1
 		}
 	};
-	int			i;
+
+	PQExpBufferData		query;
 
 	fprintf(stderr, "creating tables...\n");
 
-	for (i = 0; i < lengthof(DDLs); i++)
+	initPQExpBuffer(&query);
+
+	for (int i = 0; i < lengthof(DDLs); i++)
 	{
-		char		opts[256];
-		char		buffer[256];
 		const struct ddlinfo *ddl = &DDLs[i];
-		const char *cols;
 
 		/* Construct new create table statement. */
-		opts[0] = '\0';
+		printfPQExpBuffer(&query, "create%s table %s(%s)",
+						  unlogged_tables ? " unlogged" : "",
+						  ddl->table,
+						  (scale >= SCALE_32BIT_THRESHOLD) ? ddl->bigcols : ddl->smcols);
 
 		/* Partition pgbench_accounts table */
 		if (partition_method != PART_NONE && strcmp(ddl->table, "pgbench_accounts") == 0)
-			snprintf(opts + strlen(opts), sizeof(opts) - strlen(opts),
-					 " partition by %s (aid)", PARTITION_METHOD[partition_method]);
+			appendPQExpBuffer(&query,
+							  " partition by %s (aid)", PARTITION_METHOD[partition_method]);
 		else if (ddl->declare_fillfactor)
 			/* fillfactor is only expected on actual tables */
-			append_fillfactor(opts, sizeof(opts));
+			append_fillfactor(&query);
 
 		if (tablespace != NULL)
-		{
-			char	   *escape_tablespace;
+			append_tablespace(con, &query, "", tablespace);
 
-			escape_tablespace = PQescapeIdentifier(con, tablespace,
-												   strlen(tablespace));
-			snprintf(opts + strlen(opts), sizeof(opts) - strlen(opts),
-					 " tablespace %s", escape_tablespace);
-			PQfreemem(escape_tablespace);
-		}
-
-		cols = (scale >= SCALE_32BIT_THRESHOLD) ? ddl->bigcols : ddl->smcols;
-
-		snprintf(buffer, sizeof(buffer), "create%s table %s(%s)%s",
-				 unlogged_tables ? " unlogged" : "",
-				 ddl->table, cols, opts);
-
-		executeStatement(con, buffer);
+		executeStatement(con, query.data);
 	}
 
+	termPQExpBuffer(&query);
+
 	if (partition_method != PART_NONE)
 		createPartitions(con);
 }
@@ -3795,10 +3792,23 @@ initCreateTables(PGconn *con)
  * XXX - As default is 100, it could be removed in this case.
  */
 static void
-append_fillfactor(char *opts, int len)
+append_fillfactor(PQExpBuffer query)
 {
-	snprintf(opts + strlen(opts), len - strlen(opts),
-			 " with (fillfactor=%d)", fillfactor);
+	appendPQExpBuffer(query, " with (fillfactor=%d)", fillfactor);
+}
+
+/*
+ * add tablespace option.
+ */
+static void
+append_tablespace(PGconn *con, PQExpBuffer query, const char *kind,
+				  const char *tablespace)
+{
+	char	   *escape_tablespace;
+
+	escape_tablespace = PQescapeIdentifier(con, tablespace, strlen(tablespace));
+	appendPQExpBuffer(query, "%s tablespace %s", kind, escape_tablespace);
+	PQfreemem(escape_tablespace);
 }
 
 /*
@@ -3820,10 +3830,7 @@ initTruncateTables(PGconn *con)
 static void
 initGenerateDataClientSide(PGconn *con)
 {
-	char		sql[256];
-	PGresult   *res;
-	int			i;
-	int64		k;
+	PQExpBufferData	sql;
 
 	/* used to track elapsed time and estimate of the remaining time */
 	instr_time	start,
@@ -3846,50 +3853,47 @@ initGenerateDataClientSide(PGconn *con)
 	/* truncate away any old data */
 	initTruncateTables(con);
 
+	initPQExpBuffer(&sql);
+
 	/*
 	 * fill branches, tellers, accounts in that order in case foreign keys
 	 * already exist
 	 */
-	for (i = 0; i < nbranches * scale; i++)
+	for (int i = 0; i < nbranches * scale; i++)
 	{
 		/* "filler" column defaults to NULL */
-		snprintf(sql, sizeof(sql),
-				 "insert into pgbench_branches(bid,bbalance) values(%d,0)",
-				 i + 1);
-		executeStatement(con, sql);
+		printfPQExpBuffer(&sql,
+						  "insert into pgbench_branches(bid,bbalance) values(%d,0)",
+						  i + 1);
+		executeStatement(con, sql.data);
 	}
 
-	for (i = 0; i < ntellers * scale; i++)
+	for (int i = 0; i < ntellers * scale; i++)
 	{
 		/* "filler" column defaults to NULL */
-		snprintf(sql, sizeof(sql),
-				 "insert into pgbench_tellers(tid,bid,tbalance) values (%d,%d,0)",
-				 i + 1, i / ntellers + 1);
-		executeStatement(con, sql);
+		printfPQExpBuffer(&sql,
+						  "insert into pgbench_tellers(tid,bid,tbalance) values (%d,%d,0)",
+						  i + 1, i / ntellers + 1);
+		executeStatement(con, sql.data);
 	}
 
 	/*
 	 * accounts is big enough to be worth using COPY and tracking runtime
 	 */
-	res = PQexec(con, "copy pgbench_accounts from stdin");
-	if (PQresultStatus(res) != PGRES_COPY_IN)
-	{
-		pg_log_fatal("unexpected copy in result: %s", PQerrorMessage(con));
-		exit(1);
-	}
-	PQclear(res);
+	executeStatementExpect(con, "copy pgbench_accounts from stdin", PGRES_COPY_IN, false);
 
 	INSTR_TIME_SET_CURRENT(start);
 
-	for (k = 0; k < (int64) naccounts * scale; k++)
+	/* printf overheads should probably be avoided... */
+	for (int64 k = 0; k < (int64) naccounts * scale; k++)
 	{
 		int64		j = k + 1;
 
 		/* "filler" column defaults to blank padded empty string */
-		snprintf(sql, sizeof(sql),
-				 INT64_FORMAT "\t" INT64_FORMAT "\t%d\t\n",
-				 j, k / naccounts + 1, 0);
-		if (PQputline(con, sql))
+		printfPQExpBuffer(&sql,
+						  INT64_FORMAT "\t" INT64_FORMAT "\t%d\t\n",
+						  j, k / naccounts + 1, 0);
+		if (PQputline(con, sql.data))
 		{
 			pg_log_fatal("PQputline failed");
 			exit(1);
@@ -3951,6 +3955,8 @@ initGenerateDataClientSide(PGconn *con)
 		exit(1);
 	}
 
+	termPQExpBuffer(&sql);
+
 	executeStatement(con, "commit");
 }
 
@@ -3964,7 +3970,7 @@ initGenerateDataClientSide(PGconn *con)
 static void
 initGenerateDataServerSide(PGconn *con)
 {
-	char		sql[256];
+	PQExpBufferData	sql;
 
 	fprintf(stderr, "generating data (server-side)...\n");
 
@@ -3977,24 +3983,28 @@ initGenerateDataServerSide(PGconn *con)
 	/* truncate away any old data */
 	initTruncateTables(con);
 
-	snprintf(sql, sizeof(sql),
-			 "insert into pgbench_branches(bid,bbalance) "
-			 "select bid, 0 "
-			 "from generate_series(1, %d) as bid", nbranches * scale);
-	executeStatement(con, sql);
+	initPQExpBuffer(&sql);
 
-	snprintf(sql, sizeof(sql),
-			 "insert into pgbench_tellers(tid,bid,tbalance) "
-			 "select tid, (tid - 1) / %d + 1, 0 "
-			 "from generate_series(1, %d) as tid", ntellers, ntellers * scale);
-	executeStatement(con, sql);
+	printfPQExpBuffer(&sql,
+					  "insert into pgbench_branches(bid,bbalance) "
+					  "select bid, 0 "
+					  "from generate_series(1, %d) as bid", nbranches * scale);
+	executeStatement(con, sql.data);
 
-	snprintf(sql, sizeof(sql),
-			 "insert into pgbench_accounts(aid,bid,abalance,filler) "
-			 "select aid, (aid - 1) / %d + 1, 0, '' "
-			 "from generate_series(1, " INT64_FORMAT ") as aid",
-			 naccounts, (int64) naccounts * scale);
-	executeStatement(con, sql);
+	printfPQExpBuffer(&sql,
+					  "insert into pgbench_tellers(tid,bid,tbalance) "
+					  "select tid, (tid - 1) / %d + 1, 0 "
+					  "from generate_series(1, %d) as tid", ntellers, ntellers * scale);
+	executeStatement(con, sql.data);
+
+	printfPQExpBuffer(&sql,
+					  "insert into pgbench_accounts(aid,bid,abalance,filler) "
+					  "select aid, (aid - 1) / %d + 1, 0, '' "
+					  "from generate_series(1, "INT64_FORMAT") as aid",
+					  naccounts, (int64) naccounts * scale);
+	executeStatement(con, sql.data);
+
+	termPQExpBuffer(&sql);
 
 	executeStatement(con, "commit");
 }
@@ -4023,28 +4033,23 @@ initCreatePKeys(PGconn *con)
 		"alter table pgbench_tellers add primary key (tid)",
 		"alter table pgbench_accounts add primary key (aid)"
 	};
-	int			i;
+	PQExpBufferData query;
 
 	fprintf(stderr, "creating primary keys...\n");
-	for (i = 0; i < lengthof(DDLINDEXes); i++)
-	{
-		char		buffer[256];
+	initPQExpBuffer(&query);
 
-		strlcpy(buffer, DDLINDEXes[i], sizeof(buffer));
+	for (int i = 0; i < lengthof(DDLINDEXes); i++)
+	{
+		resetPQExpBuffer(&query);
+		appendPQExpBufferStr(&query, DDLINDEXes[i]);
 
 		if (index_tablespace != NULL)
-		{
-			char	   *escape_tablespace;
+			append_tablespace(con, &query, " using index", index_tablespace);
 
-			escape_tablespace = PQescapeIdentifier(con, index_tablespace,
-												   strlen(index_tablespace));
-			snprintf(buffer + strlen(buffer), sizeof(buffer) - strlen(buffer),
-					 " using index tablespace %s", escape_tablespace);
-			PQfreemem(escape_tablespace);
-		}
-
-		executeStatement(con, buffer);
+		executeStatement(con, query.data);
 	}
+
+	termPQExpBuffer(&query);
 }
 
 /*
#29Heikki Linnakangas
hlinnaka@iki.fi
In reply to: Fabien COELHO (#28)
Re: pgbench - refactor init functions with buffers

On 09/07/2020 10:05, Fabien COELHO wrote:

in favor of *PQExpBuffer().

Attached v7 is rebased v5 which uses PQExpBuffer, per cfbot.

Thanks! I pushed this with small changes:

- I left out the changes to executeStatement(). I'm not quite convinced
it's a good idea or worth it, and it's unrelated to the main part of
this patch, so let's handle that separately.

- I also left out changes to use the C99-style "for (int i = 0; ...)"
construct. I think that's a good change for readability, but again
unrelated to this and hardly worth changing existing code for.

- I inlined the append_tablespace() function back to the callers. And I
did the same to the append_fillfactor() function, too. It seems more
readable to just call appendPQExpBuffer() diretly, than encapulate the
single appendPQExpBuffer() call in a helper function.

@@ -3880,15 +3868,16 @@ initGenerateDataClientSide(PGconn *con)

INSTR_TIME_SET_CURRENT(start);

+ /* printf overheads should probably be avoided... */
for (k = 0; k < (int64) naccounts * scale; k++)
{
int64 j = k + 1;

/* "filler" column defaults to blank padded empty string */
-		snprintf(sql, sizeof(sql),
-				 INT64_FORMAT "\t" INT64_FORMAT "\t%d\t\n",
-				 j, k / naccounts + 1, 0);
-		if (PQputline(con, sql))
+		printfPQExpBuffer(&sql,
+						  INT64_FORMAT "\t" INT64_FORMAT "\t%d\t\n",
+						  j, k / naccounts + 1, 0);
+		if (PQputline(con, sql.data))
{
pg_log_fatal("PQputline failed");
exit(1);

Can you elaborate what you meant by the new "print overheads should
probably be avoided" comment? I left that out since it seems unrelated
to switching to PQExpBuffer.

- Heikki

#30Fabien COELHO
coelho@cri.ensmp.fr
In reply to: Heikki Linnakangas (#29)
Re: pgbench - refactor init functions with buffers

Can you elaborate what you meant by the new "print overheads should probably
be avoided" comment?

Because printf is slow and this is on the critical path of data
generation. Printf has to interpret the format each time just to print
three ints, specialized functions could be used which would allow to skip
the repeated format parsing.

I left that out since it seems unrelated to switching to PQExpBuffer.

Yep.

Thanks for the commit. Getting rid of most snprintf is a relief.

--
Fabien.