From 92e32881fd535c9e3292aae68f3331057d633492 Mon Sep 17 00:00:00 2001 From: Andres Freund Date: Wed, 13 Sep 2017 18:39:24 -0700 Subject: [PATCH 3/6] Add more efficient functions to pqformat API. There's three prongs to achieve greater efficiency here: 1) Allow reusing a stringbuffer across pq_beginmessage/endmessage, with the new pq_beginmessage_reuse/endmessage_reuse. This can be beneficial both because it avoids allocating the initial buffer, and because it's more likely to already have an correctly sized buffer. 2) Replacing pq_sendint() with pq_sendint$width() inline functions. Previously unnecessary and unpredictable branches in pq_sendint() were needed. Additionally the replacement functions are implemented more efficiently. pq_sendint is now deprecated, a separate commit will convert all in-tree callers. 3) Add pq_writeint$width(), pq_writestring(). These rely on sufficient space in the StringInfo's buffer, avoiding individual space checks & potential individual resizing. To allow this to be used for strings, expose mbutil.c's MAX_CONVERSION_GROWTH. Followup commits will make use of these facilities. Author: Andres Freund Discussion: https://postgr.es/m/20170914063418.sckdzgjfrsbekae4@alap3.anarazel.de --- src/backend/libpq/pqformat.c | 88 ++++++++------------- src/backend/utils/mb/mbutils.c | 11 --- src/include/libpq/pqformat.h | 168 ++++++++++++++++++++++++++++++++++++++++- src/include/mb/pg_wchar.h | 11 +++ 4 files changed, 208 insertions(+), 70 deletions(-) diff --git a/src/backend/libpq/pqformat.c b/src/backend/libpq/pqformat.c index 2414d0d8e9a..a5698390ae7 100644 --- a/src/backend/libpq/pqformat.c +++ b/src/backend/libpq/pqformat.c @@ -97,13 +97,24 @@ pq_beginmessage(StringInfo buf, char msgtype) } /* -------------------------------- - * pq_sendbyte - append a raw byte to a StringInfo buffer + + * pq_beginmessage_reuse - initialize for sending a message, reuse buffer + * + * This requires the buffer to be allocated in an sufficiently long-lived + * memory context. * -------------------------------- */ void -pq_sendbyte(StringInfo buf, int byt) +pq_beginmessage_reuse(StringInfo buf, char msgtype) { - appendStringInfoCharMacro(buf, byt); + resetStringInfo(buf); + + /* + * We stash the message type into the buffer's cursor field, expecting + * that the pq_sendXXX routines won't touch it. We could alternatively + * make it the first byte of the buffer contents, but this seems easier. + */ + buf->cursor = msgtype; } /* -------------------------------- @@ -113,6 +124,7 @@ pq_sendbyte(StringInfo buf, int byt) void pq_sendbytes(StringInfo buf, const char *data, int datalen) { + /* use variant that maintains a trailing null-byte, out of caution */ appendBinaryStringInfo(buf, data, datalen); } @@ -137,13 +149,13 @@ pq_sendcountedtext(StringInfo buf, const char *str, int slen, if (p != str) /* actual conversion has been done? */ { slen = strlen(p); - pq_sendint(buf, slen + extra, 4); + pq_sendint32(buf, slen + extra); appendBinaryStringInfoNT(buf, p, slen); pfree(p); } else { - pq_sendint(buf, slen + extra, 4); + pq_sendint32(buf, slen + extra); appendBinaryStringInfoNT(buf, str, slen); } } @@ -227,53 +239,6 @@ pq_send_ascii_string(StringInfo buf, const char *str) appendStringInfoChar(buf, '\0'); } -/* -------------------------------- - * pq_sendint - append a binary integer to a StringInfo buffer - * -------------------------------- - */ -void -pq_sendint(StringInfo buf, int i, int b) -{ - unsigned char n8; - uint16 n16; - uint32 n32; - - switch (b) - { - case 1: - n8 = (unsigned char) i; - appendBinaryStringInfoNT(buf, (char *) &n8, 1); - break; - case 2: - n16 = pg_hton16((uint16) i); - appendBinaryStringInfoNT(buf, (char *) &n16, 2); - break; - case 4: - n32 = pg_hton32((uint32) i); - appendBinaryStringInfoNT(buf, (char *) &n32, 4); - break; - default: - elog(ERROR, "unsupported integer size %d", b); - break; - } -} - -/* -------------------------------- - * pq_sendint64 - append a binary 8-byte int to a StringInfo buffer - * - * It is tempting to merge this with pq_sendint, but we'd have to make the - * argument int64 for all data widths --- that could be a big performance - * hit on machines where int64 isn't efficient. - * -------------------------------- - */ -void -pq_sendint64(StringInfo buf, int64 i) -{ - uint64 n64 = pg_hton64(i); - - appendBinaryStringInfoNT(buf, (char *) &n64, sizeof(n64)); -} - /* -------------------------------- * pq_sendfloat4 - append a float4 to a StringInfo buffer * @@ -295,9 +260,7 @@ pq_sendfloat4(StringInfo buf, float4 f) } swap; swap.f = f; - swap.i = pg_hton32(swap.i); - - appendBinaryStringInfoNT(buf, (char *) &swap.i, 4); + pq_sendint32(buf, swap.i); } /* -------------------------------- @@ -341,6 +304,21 @@ pq_endmessage(StringInfo buf) buf->data = NULL; } +/* -------------------------------- + * pq_endmessage_reuse - send the completed message to the frontend + * + * The data buffer is *not* freed, allowing to reuse the buffer with + * pg_beginmessage_reuse. + -------------------------------- + */ + +void +pq_endmessage_reuse(StringInfo buf) +{ + /* msgtype was saved in cursor field */ + (void) pq_putmessage(buf->cursor, buf->data, buf->len); +} + /* -------------------------------- * pq_begintypsend - initialize for constructing a bytea result diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c index c4fbe0903ba..56f4dc1453c 100644 --- a/src/backend/utils/mb/mbutils.c +++ b/src/backend/utils/mb/mbutils.c @@ -41,17 +41,6 @@ #include "utils/memutils.h" #include "utils/syscache.h" -/* - * When converting strings between different encodings, we assume that space - * for converted result is 4-to-1 growth in the worst case. The rate for - * currently supported encoding pairs are within 3 (SJIS JIS X0201 half width - * kanna -> UTF8 is the worst case). So "4" should be enough for the moment. - * - * Note that this is not the same as the maximum character width in any - * particular encoding. - */ -#define MAX_CONVERSION_GROWTH 4 - /* * We maintain a simple linked list caching the fmgr lookup info for the * currently selected conversion functions, as well as any that have been diff --git a/src/include/libpq/pqformat.h b/src/include/libpq/pqformat.h index 32112547a0b..9a546b48915 100644 --- a/src/include/libpq/pqformat.h +++ b/src/include/libpq/pqformat.h @@ -14,20 +14,180 @@ #define PQFORMAT_H #include "lib/stringinfo.h" +#include "mb/pg_wchar.h" +#include "port/pg_bswap.h" extern void pq_beginmessage(StringInfo buf, char msgtype); -extern void pq_sendbyte(StringInfo buf, int byt); +extern void pq_beginmessage_reuse(StringInfo buf, char msgtype); +extern void pq_endmessage(StringInfo buf); +extern void pq_endmessage_reuse(StringInfo buf); + extern void pq_sendbytes(StringInfo buf, const char *data, int datalen); extern void pq_sendcountedtext(StringInfo buf, const char *str, int slen, bool countincludesself); extern void pq_sendtext(StringInfo buf, const char *str, int slen); extern void pq_sendstring(StringInfo buf, const char *str); extern void pq_send_ascii_string(StringInfo buf, const char *str); -extern void pq_sendint(StringInfo buf, int i, int b); -extern void pq_sendint64(StringInfo buf, int64 i); extern void pq_sendfloat4(StringInfo buf, float4 f); extern void pq_sendfloat8(StringInfo buf, float8 f); -extern void pq_endmessage(StringInfo buf); + +extern void pq_sendfloat4(StringInfo buf, float4 f); +extern void pq_sendfloat8(StringInfo buf, float8 f); + +/* + * Append a int8 to a StringInfo buffer, which already has enough space + * preallocated. + * + * The use of restrict allows the compiler to optimize the code based on the + * assumption that buf, buf->len, buf->data and *buf->data don't + * overlap. Without the annotation buf->len etc cannot be kept in a register + * over subsequent pq_writeint* calls. + */ +static inline void +pq_writeint8(StringInfo restrict buf, int8 i) +{ + int8 ni = i; + + Assert(buf->len + sizeof(i) <= buf->maxlen); + memcpy((char *restrict) (buf->data + buf->len), &ni, sizeof(ni)); + buf->len += sizeof(i); +} + +/* + * Append a int16 to a StringInfo buffer, which already has enough space + * preallocated. + */ +static inline void +pq_writeint16(StringInfo restrict buf, int16 i) +{ + int16 ni = pg_hton16(i); + + Assert(buf->len + sizeof(ni) <= buf->maxlen); + memcpy((char *restrict) (buf->data + buf->len), &ni, sizeof(i)); + buf->len += sizeof(i); +} + +/* + * Append a int32 to a StringInfo buffer, which already has enough space + * preallocated. + */ +static inline void +pq_writeint32(StringInfo restrict buf, int32 i) +{ + int32 ni = pg_hton32(i); + + Assert(buf->len + sizeof(i) <= buf->maxlen); + memcpy((char *restrict) (buf->data + buf->len), &ni, sizeof(i)); + buf->len += sizeof(i); +} + +/* + * Append a int64 to a StringInfo buffer, which already has enough space + * preallocated. + */ +static inline void +pq_writeint64(StringInfo restrict buf, int64 i) +{ + int64 ni = pg_hton64(i); + + Assert(buf->len + sizeof(i) <= buf->maxlen); + memcpy((char *restrict) (buf->data + buf->len), &ni, sizeof(i)); + buf->len += sizeof(i); +} + +/* + * Append a null-terminated text string (with conversion) to a buffer with + * preallocated space. + * + * NB: The pre-allocated space needs to be sufficient for the string after + * converting to client encoding. + * + * NB: passed text string must be null-terminated, and so is the data + * sent to the frontend. + */ +static inline void +pq_writestring(StringInfo restrict buf, const char *restrict str) +{ + int slen = strlen(str); + char *p; + + p = pg_server_to_client(str, slen); + if (p != str) /* actual conversion has been done? */ + slen = strlen(p); + + Assert(buf->len + slen + 1 <= buf->maxlen); + + memcpy(((char *restrict) buf->data + buf->len), p, slen + 1); + buf->len += slen + 1; + + if (p != str) + pfree(p); +} + +/* append a binary int8 to a StringInfo buffer */ +static inline void +pq_sendint8(StringInfo buf, int8 i) +{ + enlargeStringInfo(buf, sizeof(i)); + pq_writeint8(buf, i); +} + +/* append a binary int16 to a StringInfo buffer */ +static inline void +pq_sendint16(StringInfo buf, int16 i) +{ + enlargeStringInfo(buf, sizeof(i)); + pq_writeint16(buf, i); +} + +/* append a binary int32 to a StringInfo buffer */ +static inline void +pq_sendint32(StringInfo buf, int32 i) +{ + enlargeStringInfo(buf, sizeof(i)); + pq_writeint32(buf, i); +} + +/* append a binary int64 to a StringInfo buffer */ +static inline void +pq_sendint64(StringInfo buf, int64 i) +{ + enlargeStringInfo(buf, sizeof(i)); + pq_writeint64(buf, i); +} + +/* append a binary byte to a StringInfo buffer */ +static inline void +pq_sendbyte(StringInfo buf, int8 byt) +{ + pq_sendint8(buf, byt); +} + +/* + * Append a binary integer to a StringInfo buffer + * + * This function is deprecated. + */ +static inline void +pq_sendint(StringInfo buf, int i, int b) +{ + switch (b) + { + case 1: + pq_sendint8(buf, (int8) i); + break; + case 2: + pq_sendint16(buf, (int16) i); + break; + case 4: + pq_sendint32(buf, (int32) i); + break; + default: + elog(ERROR, "unsupported integer size %d", b); + break; + } +} + extern void pq_begintypsend(StringInfo buf); extern bytea *pq_endtypsend(StringInfo buf); diff --git a/src/include/mb/pg_wchar.h b/src/include/mb/pg_wchar.h index d57ef017cb4..9227d634f66 100644 --- a/src/include/mb/pg_wchar.h +++ b/src/include/mb/pg_wchar.h @@ -304,6 +304,17 @@ typedef enum pg_enc /* On FE are possible all encodings */ #define PG_VALID_FE_ENCODING(_enc) PG_VALID_ENCODING(_enc) +/* + * When converting strings between different encodings, we assume that space + * for converted result is 4-to-1 growth in the worst case. The rate for + * currently supported encoding pairs are within 3 (SJIS JIS X0201 half width + * kanna -> UTF8 is the worst case). So "4" should be enough for the moment. + * + * Note that this is not the same as the maximum character width in any + * particular encoding. + */ +#define MAX_CONVERSION_GROWTH 4 + /* * Table for mapping an encoding number to official encoding name and * possibly other subsidiary data. Be careful to check encoding number -- 2.14.1.536.g6867272d5b.dirty