Python 3.1 support
Here's the patch to support Python >=3.1 with PL/Python. The
compatibility code is mostly in line with the usual 2->3 C porting
practice and is documented inline.
I needed to create an arguably weird hack to manage the regression
tests. Instead of creating a new expected file for pretty much every
test file and also for some input files (where Python syntax had
changed), a sed script creates a complete Python 3 compatible set of
input and output files. Doesn't look pretty but works quite well. If
anyone has a better idea, please let me know.
Attachments:
python3.1.patchtext/x-patch; charset=UTF-8; name=python3.1.patchDownload
diff --git a/src/pl/plpython/Makefile b/src/pl/plpython/Makefile
index 373bc79..840b874 100644
--- a/src/pl/plpython/Makefile
+++ b/src/pl/plpython/Makefile
@@ -88,8 +88,32 @@ installdirs: installdirs-lib
uninstall: uninstall-lib
+ifneq (,$(findstring 3.,$(python_version)))
+# Adjust regression tests for Python 3 compatibility
+prep3:
+ mkdir -p 3 3/sql 3/expected
+ for file in $(srcdir)/sql/* $(srcdir)/expected/*; do \
+ sed -r -e 's/except ([[:alpha:].]+), ?(\w+):/except \1 as \2:/g' \
+ -e "s/<type 'exceptions\.(\w+)'>/<class '\1'>/g" \
+ -e "s/<type 'long'>/<class 'int'>/g" \
+ -e "s/\b([0-9]+)L\b/\1/g" \
+ -e 's/\bu"/"/g' \
+ -e "s/\bu'/'/g" \
+ -e "s/def next\b/def __next__/g" \
+ $$file >`echo $$file | sed 's,$(srcdir),3,'`; \
+ done
+
+clean3:
+ rm -rf 3/
+
+installcheck: submake prep3
+ cd 3 && $(top_builddir)/../src/test/regress/pg_regress --inputdir=. --psqldir=$(PSQLDIR) $(REGRESS_OPTS) $(REGRESS)
+
+clean: clean3
+else
installcheck: submake
$(top_builddir)/src/test/regress/pg_regress --inputdir=$(srcdir) --psqldir=$(PSQLDIR) $(REGRESS_OPTS) $(REGRESS)
+endif
.PHONY: submake
submake:
diff --git a/src/pl/plpython/expected/README b/src/pl/plpython/expected/README
index 47f31e8..a187937 100644
--- a/src/pl/plpython/expected/README
+++ b/src/pl/plpython/expected/README
@@ -8,3 +8,5 @@ plpython_unicode_0.out any version, when server encoding != SQL_ASCII and clien
plpython_unicode_2.out Python 2.2
plpython_unicode_3.out Python 2.3, 2.4
plpython_unicode_5.out Python 2.5, 2.6
+
+plpython_types_3.out Python 3.1
diff --git a/src/pl/plpython/expected/plpython_types_3.out b/src/pl/plpython/expected/plpython_types_3.out
new file mode 100644
index 0000000..3fcb0f4
--- /dev/null
+++ b/src/pl/plpython/expected/plpython_types_3.out
@@ -0,0 +1,479 @@
+--
+-- Test data type behavior
+--
+--
+-- Base/common types
+--
+CREATE FUNCTION test_type_conversion_bool(x bool) RETURNS bool AS $$
+plpy.info(x, type(x))
+return x
+$$ LANGUAGE plpythonu;
+SELECT * FROM test_type_conversion_bool(true);
+INFO: (True, <class 'bool'>)
+CONTEXT: PL/Python function "test_type_conversion_bool"
+ test_type_conversion_bool
+---------------------------
+ t
+(1 row)
+
+SELECT * FROM test_type_conversion_bool(false);
+INFO: (False, <class 'bool'>)
+CONTEXT: PL/Python function "test_type_conversion_bool"
+ test_type_conversion_bool
+---------------------------
+ f
+(1 row)
+
+SELECT * FROM test_type_conversion_bool(null);
+INFO: (None, <class 'NoneType'>)
+CONTEXT: PL/Python function "test_type_conversion_bool"
+ test_type_conversion_bool
+---------------------------
+
+(1 row)
+
+-- test various other ways to express Booleans in Python
+CREATE FUNCTION test_type_conversion_bool_other(n int) RETURNS bool AS $$
+# numbers
+if n == 0:
+ ret = 0
+elif n == 1:
+ ret = 5
+# strings
+elif n == 2:
+ ret = ''
+elif n == 3:
+ ret = 'fa' # true in Python, false in PostgreSQL
+# containers
+elif n == 4:
+ ret = []
+elif n == 5:
+ ret = [0]
+plpy.info(ret, not not ret)
+return ret
+$$ LANGUAGE plpythonu;
+SELECT * FROM test_type_conversion_bool_other(0);
+INFO: (0, False)
+CONTEXT: PL/Python function "test_type_conversion_bool_other"
+ test_type_conversion_bool_other
+---------------------------------
+ f
+(1 row)
+
+SELECT * FROM test_type_conversion_bool_other(1);
+INFO: (5, True)
+CONTEXT: PL/Python function "test_type_conversion_bool_other"
+ test_type_conversion_bool_other
+---------------------------------
+ t
+(1 row)
+
+SELECT * FROM test_type_conversion_bool_other(2);
+INFO: ('', False)
+CONTEXT: PL/Python function "test_type_conversion_bool_other"
+ test_type_conversion_bool_other
+---------------------------------
+ f
+(1 row)
+
+SELECT * FROM test_type_conversion_bool_other(3);
+INFO: ('fa', True)
+CONTEXT: PL/Python function "test_type_conversion_bool_other"
+ test_type_conversion_bool_other
+---------------------------------
+ t
+(1 row)
+
+SELECT * FROM test_type_conversion_bool_other(4);
+INFO: ([], False)
+CONTEXT: PL/Python function "test_type_conversion_bool_other"
+ test_type_conversion_bool_other
+---------------------------------
+ f
+(1 row)
+
+SELECT * FROM test_type_conversion_bool_other(5);
+INFO: ([0], True)
+CONTEXT: PL/Python function "test_type_conversion_bool_other"
+ test_type_conversion_bool_other
+---------------------------------
+ t
+(1 row)
+
+CREATE FUNCTION test_type_conversion_char(x char) RETURNS char AS $$
+plpy.info(x, type(x))
+return x
+$$ LANGUAGE plpythonu;
+SELECT * FROM test_type_conversion_char('a');
+INFO: ('a', <class 'str'>)
+CONTEXT: PL/Python function "test_type_conversion_char"
+ test_type_conversion_char
+---------------------------
+ a
+(1 row)
+
+SELECT * FROM test_type_conversion_char(null);
+INFO: (None, <class 'NoneType'>)
+CONTEXT: PL/Python function "test_type_conversion_char"
+ test_type_conversion_char
+---------------------------
+
+(1 row)
+
+CREATE FUNCTION test_type_conversion_int2(x int2) RETURNS int2 AS $$
+plpy.info(x, type(x))
+return x
+$$ LANGUAGE plpythonu;
+SELECT * FROM test_type_conversion_int2(100::int2);
+INFO: (100, <class 'int'>)
+CONTEXT: PL/Python function "test_type_conversion_int2"
+ test_type_conversion_int2
+---------------------------
+ 100
+(1 row)
+
+SELECT * FROM test_type_conversion_int2(-100::int2);
+INFO: (-100, <class 'int'>)
+CONTEXT: PL/Python function "test_type_conversion_int2"
+ test_type_conversion_int2
+---------------------------
+ -100
+(1 row)
+
+SELECT * FROM test_type_conversion_int2(null);
+INFO: (None, <class 'NoneType'>)
+CONTEXT: PL/Python function "test_type_conversion_int2"
+ test_type_conversion_int2
+---------------------------
+
+(1 row)
+
+CREATE FUNCTION test_type_conversion_int4(x int4) RETURNS int4 AS $$
+plpy.info(x, type(x))
+return x
+$$ LANGUAGE plpythonu;
+SELECT * FROM test_type_conversion_int4(100);
+INFO: (100, <class 'int'>)
+CONTEXT: PL/Python function "test_type_conversion_int4"
+ test_type_conversion_int4
+---------------------------
+ 100
+(1 row)
+
+SELECT * FROM test_type_conversion_int4(-100);
+INFO: (-100, <class 'int'>)
+CONTEXT: PL/Python function "test_type_conversion_int4"
+ test_type_conversion_int4
+---------------------------
+ -100
+(1 row)
+
+SELECT * FROM test_type_conversion_int4(null);
+INFO: (None, <class 'NoneType'>)
+CONTEXT: PL/Python function "test_type_conversion_int4"
+ test_type_conversion_int4
+---------------------------
+
+(1 row)
+
+CREATE FUNCTION test_type_conversion_int8(x int8) RETURNS int8 AS $$
+plpy.info(x, type(x))
+return x
+$$ LANGUAGE plpythonu;
+SELECT * FROM test_type_conversion_int8(100);
+INFO: (100, <class 'int'>)
+CONTEXT: PL/Python function "test_type_conversion_int8"
+ test_type_conversion_int8
+---------------------------
+ 100
+(1 row)
+
+SELECT * FROM test_type_conversion_int8(-100);
+INFO: (-100, <class 'int'>)
+CONTEXT: PL/Python function "test_type_conversion_int8"
+ test_type_conversion_int8
+---------------------------
+ -100
+(1 row)
+
+SELECT * FROM test_type_conversion_int8(5000000000);
+INFO: (5000000000, <class 'int'>)
+CONTEXT: PL/Python function "test_type_conversion_int8"
+ test_type_conversion_int8
+---------------------------
+ 5000000000
+(1 row)
+
+SELECT * FROM test_type_conversion_int8(null);
+INFO: (None, <class 'NoneType'>)
+CONTEXT: PL/Python function "test_type_conversion_int8"
+ test_type_conversion_int8
+---------------------------
+
+(1 row)
+
+CREATE FUNCTION test_type_conversion_numeric(x numeric) RETURNS numeric AS $$
+plpy.info(x, type(x))
+return x
+$$ LANGUAGE plpythonu;
+/* The current implementation converts numeric to float. */
+SELECT * FROM test_type_conversion_numeric(100);
+INFO: (100.0, <class 'float'>)
+CONTEXT: PL/Python function "test_type_conversion_numeric"
+ test_type_conversion_numeric
+------------------------------
+ 100.0
+(1 row)
+
+SELECT * FROM test_type_conversion_numeric(-100);
+INFO: (-100.0, <class 'float'>)
+CONTEXT: PL/Python function "test_type_conversion_numeric"
+ test_type_conversion_numeric
+------------------------------
+ -100.0
+(1 row)
+
+SELECT * FROM test_type_conversion_numeric(5000000000.5);
+INFO: (5000000000.5, <class 'float'>)
+CONTEXT: PL/Python function "test_type_conversion_numeric"
+ test_type_conversion_numeric
+------------------------------
+ 5000000000.5
+(1 row)
+
+SELECT * FROM test_type_conversion_numeric(null);
+INFO: (None, <class 'NoneType'>)
+CONTEXT: PL/Python function "test_type_conversion_numeric"
+ test_type_conversion_numeric
+------------------------------
+
+(1 row)
+
+CREATE FUNCTION test_type_conversion_float4(x float4) RETURNS float4 AS $$
+plpy.info(x, type(x))
+return x
+$$ LANGUAGE plpythonu;
+SELECT * FROM test_type_conversion_float4(100);
+INFO: (100.0, <class 'float'>)
+CONTEXT: PL/Python function "test_type_conversion_float4"
+ test_type_conversion_float4
+-----------------------------
+ 100
+(1 row)
+
+SELECT * FROM test_type_conversion_float4(-100);
+INFO: (-100.0, <class 'float'>)
+CONTEXT: PL/Python function "test_type_conversion_float4"
+ test_type_conversion_float4
+-----------------------------
+ -100
+(1 row)
+
+SELECT * FROM test_type_conversion_float4(5000.5);
+INFO: (5000.5, <class 'float'>)
+CONTEXT: PL/Python function "test_type_conversion_float4"
+ test_type_conversion_float4
+-----------------------------
+ 5000.5
+(1 row)
+
+SELECT * FROM test_type_conversion_float4(null);
+INFO: (None, <class 'NoneType'>)
+CONTEXT: PL/Python function "test_type_conversion_float4"
+ test_type_conversion_float4
+-----------------------------
+
+(1 row)
+
+CREATE FUNCTION test_type_conversion_float8(x float8) RETURNS float8 AS $$
+plpy.info(x, type(x))
+return x
+$$ LANGUAGE plpythonu;
+SELECT * FROM test_type_conversion_float8(100);
+INFO: (100.0, <class 'float'>)
+CONTEXT: PL/Python function "test_type_conversion_float8"
+ test_type_conversion_float8
+-----------------------------
+ 100
+(1 row)
+
+SELECT * FROM test_type_conversion_float8(-100);
+INFO: (-100.0, <class 'float'>)
+CONTEXT: PL/Python function "test_type_conversion_float8"
+ test_type_conversion_float8
+-----------------------------
+ -100
+(1 row)
+
+SELECT * FROM test_type_conversion_float8(5000000000.5);
+INFO: (5000000000.5, <class 'float'>)
+CONTEXT: PL/Python function "test_type_conversion_float8"
+ test_type_conversion_float8
+-----------------------------
+ 5000000000.5
+(1 row)
+
+SELECT * FROM test_type_conversion_float8(null);
+INFO: (None, <class 'NoneType'>)
+CONTEXT: PL/Python function "test_type_conversion_float8"
+ test_type_conversion_float8
+-----------------------------
+
+(1 row)
+
+CREATE FUNCTION test_type_conversion_text(x text) RETURNS text AS $$
+plpy.info(x, type(x))
+return x
+$$ LANGUAGE plpythonu;
+SELECT * FROM test_type_conversion_text('hello world');
+INFO: ('hello world', <class 'str'>)
+CONTEXT: PL/Python function "test_type_conversion_text"
+ test_type_conversion_text
+---------------------------
+ hello world
+(1 row)
+
+SELECT * FROM test_type_conversion_text(null);
+INFO: (None, <class 'NoneType'>)
+CONTEXT: PL/Python function "test_type_conversion_text"
+ test_type_conversion_text
+---------------------------
+
+(1 row)
+
+CREATE FUNCTION test_type_conversion_bytea(x bytea) RETURNS bytea AS $$
+plpy.info(x, type(x))
+return x
+$$ LANGUAGE plpythonu;
+SELECT * FROM test_type_conversion_bytea('hello world');
+INFO: (b'hello world', <class 'bytes'>)
+CONTEXT: PL/Python function "test_type_conversion_bytea"
+ test_type_conversion_bytea
+----------------------------
+ \x68656c6c6f20776f726c64
+(1 row)
+
+SELECT * FROM test_type_conversion_bytea(E'null\\000byte');
+INFO: (b'null\x00byte', <class 'bytes'>)
+CONTEXT: PL/Python function "test_type_conversion_bytea"
+ test_type_conversion_bytea
+----------------------------
+ \x6e756c6c0062797465
+(1 row)
+
+SELECT * FROM test_type_conversion_bytea(null);
+INFO: (None, <class 'NoneType'>)
+CONTEXT: PL/Python function "test_type_conversion_bytea"
+ test_type_conversion_bytea
+----------------------------
+
+(1 row)
+
+CREATE FUNCTION test_type_marshal() RETURNS bytea AS $$
+import marshal
+return marshal.dumps('hello world')
+$$ LANGUAGE plpythonu;
+CREATE FUNCTION test_type_unmarshal(x bytea) RETURNS text AS $$
+import marshal
+try:
+ return marshal.loads(x)
+except ValueError as e:
+ return 'FAILED: ' + str(e)
+$$ LANGUAGE plpythonu;
+SELECT test_type_unmarshal(x) FROM test_type_marshal() x;
+ test_type_unmarshal
+---------------------
+ hello world
+(1 row)
+
+--
+-- Domains
+--
+CREATE DOMAIN booltrue AS bool CHECK (VALUE IS TRUE OR VALUE IS NULL);
+CREATE FUNCTION test_type_conversion_booltrue(x booltrue, y bool) RETURNS booltrue AS $$
+return y
+$$ LANGUAGE plpythonu;
+SELECT * FROM test_type_conversion_booltrue(true, true);
+ test_type_conversion_booltrue
+-------------------------------
+ t
+(1 row)
+
+SELECT * FROM test_type_conversion_booltrue(false, true);
+ERROR: value for domain booltrue violates check constraint "booltrue_check"
+SELECT * FROM test_type_conversion_booltrue(true, false);
+ERROR: value for domain booltrue violates check constraint "booltrue_check"
+CONTEXT: while creating return value
+PL/Python function "test_type_conversion_booltrue"
+CREATE DOMAIN uint2 AS int2 CHECK (VALUE >= 0);
+CREATE FUNCTION test_type_conversion_uint2(x uint2, y int) RETURNS uint2 AS $$
+plpy.info(x, type(x))
+return y
+$$ LANGUAGE plpythonu;
+SELECT * FROM test_type_conversion_uint2(100::uint2, 50);
+INFO: (100, <class 'int'>)
+CONTEXT: PL/Python function "test_type_conversion_uint2"
+ test_type_conversion_uint2
+----------------------------
+ 50
+(1 row)
+
+SELECT * FROM test_type_conversion_uint2(100::uint2, -50);
+INFO: (100, <class 'int'>)
+CONTEXT: PL/Python function "test_type_conversion_uint2"
+ERROR: value for domain uint2 violates check constraint "uint2_check"
+CONTEXT: while creating return value
+PL/Python function "test_type_conversion_uint2"
+SELECT * FROM test_type_conversion_uint2(null, 1);
+INFO: (None, <class 'NoneType'>)
+CONTEXT: PL/Python function "test_type_conversion_uint2"
+ test_type_conversion_uint2
+----------------------------
+ 1
+(1 row)
+
+CREATE DOMAIN nnint AS int CHECK (VALUE IS NOT NULL);
+CREATE FUNCTION test_type_conversion_nnint(x nnint, y int) RETURNS nnint AS $$
+return y
+$$ LANGUAGE plpythonu;
+SELECT * FROM test_type_conversion_nnint(10, 20);
+ test_type_conversion_nnint
+----------------------------
+ 20
+(1 row)
+
+SELECT * FROM test_type_conversion_nnint(null, 20);
+ERROR: value for domain nnint violates check constraint "nnint_check"
+SELECT * FROM test_type_conversion_nnint(10, null);
+ERROR: value for domain nnint violates check constraint "nnint_check"
+CONTEXT: while creating return value
+PL/Python function "test_type_conversion_nnint"
+CREATE DOMAIN bytea10 AS bytea CHECK (octet_length(VALUE) = 10 AND VALUE IS NOT NULL);
+CREATE FUNCTION test_type_conversion_bytea10(x bytea10, y bytea) RETURNS bytea10 AS $$
+plpy.info(x, type(x))
+return y
+$$ LANGUAGE plpythonu;
+SELECT * FROM test_type_conversion_bytea10('hello wold', 'hello wold');
+INFO: (b'hello wold', <class 'bytes'>)
+CONTEXT: PL/Python function "test_type_conversion_bytea10"
+ test_type_conversion_bytea10
+------------------------------
+ \x68656c6c6f20776f6c64
+(1 row)
+
+SELECT * FROM test_type_conversion_bytea10('hello world', 'hello wold');
+ERROR: value for domain bytea10 violates check constraint "bytea10_check"
+SELECT * FROM test_type_conversion_bytea10('hello word', 'hello world');
+INFO: (b'hello word', <class 'bytes'>)
+CONTEXT: PL/Python function "test_type_conversion_bytea10"
+ERROR: value for domain bytea10 violates check constraint "bytea10_check"
+CONTEXT: while creating return value
+PL/Python function "test_type_conversion_bytea10"
+SELECT * FROM test_type_conversion_bytea10(null, 'hello word');
+ERROR: value for domain bytea10 violates check constraint "bytea10_check"
+SELECT * FROM test_type_conversion_bytea10('hello word', null);
+INFO: (b'hello word', <class 'bytes'>)
+CONTEXT: PL/Python function "test_type_conversion_bytea10"
+ERROR: value for domain bytea10 violates check constraint "bytea10_check"
+CONTEXT: while creating return value
+PL/Python function "test_type_conversion_bytea10"
diff --git a/src/pl/plpython/plpython.c b/src/pl/plpython/plpython.c
index 6fd4aca..0d6c261 100644
--- a/src/pl/plpython/plpython.c
+++ b/src/pl/plpython/plpython.c
@@ -40,6 +40,48 @@ typedef int Py_ssize_t;
#define PyBool_FromLong(x) PyInt_FromLong(x)
#endif
+/*
+ * Python 2/3 strings/unicode/bytes handling. Python 2 has strings
+ * and unicode, Python 3 has strings, which are unicode on the C
+ * level, and bytes. The porting convention, which is similarly used
+ * in Python 2.6, is that "Unicode" is always unicode, and "Bytes" are
+ * bytes in Python 3 and strings in Python 2. Since we keep
+ * supporting Python 2 and its usual strings, we provide a
+ * compatibility layer for Python 3 that when asked to convert a C
+ * string to a Python string it converts the C string from the
+ * PostgreSQL server encoding to a Python Unicode object.
+ */
+
+#if PY_VERSION_HEX < 0x02060000
+/* This is exactly the compatibility layer that Python 2.6 uses. */
+#define PyBytes_AsString PyString_AsString
+#define PyBytes_FromStringAndSize PyString_FromStringAndSize
+#define PyBytes_Size PyString_Size
+#define PyObject_Bytes PyObject_Str
+#endif
+
+#if PY_MAJOR_VERSION >= 3
+#define PyString_Check(x) 0
+#define PyString_AsString(x) PLyUnicode_AsString(x)
+#define PyString_FromString(x) PLyUnicode_FromString(x)
+#endif
+
+/*
+ * Python 3 only has long.
+ */
+#if PY_MAJOR_VERSION >= 3
+#define PyInt_FromLong(x) PyLong_FromLong(x)
+#endif
+
+/*
+ * PyVarObject_HEAD_INIT was added in Python 2.6. Its use is
+ * necessary to handle both Python 2 and 3. This replacement
+ * definition is for Python <=2.5
+ */
+#ifndef PyVarObject_HEAD_INIT
+#define PyVarObject_HEAD_INIT(type, size) \
+ PyObject_HEAD_INIT(type) size,
+#endif
#include "postgres.h"
@@ -240,7 +282,11 @@ static char *PLy_strdup(const char *);
static void PLy_free(void *);
static PyObject*PLyUnicode_Str(PyObject *unicode);
+static PyObject*PLyUnicode_Bytes(PyObject *unicode);
static char *PLyUnicode_AsString(PyObject *unicode);
+#if PY_MAJOR_VERSION >= 3
+static PyObject *PLyUnicode_FromString(const char *s);
+#endif
/* sub handlers for functions and triggers */
static Datum PLy_function_handler(FunctionCallInfo fcinfo, PLyProcedure *);
@@ -282,7 +328,7 @@ static PyObject *PLyFloat_FromNumeric(PLyDatumToOb *arg, Datum d);
static PyObject *PLyInt_FromInt16(PLyDatumToOb *arg, Datum d);
static PyObject *PLyInt_FromInt32(PLyDatumToOb *arg, Datum d);
static PyObject *PLyLong_FromInt64(PLyDatumToOb *arg, Datum d);
-static PyObject *PLyString_FromBytea(PLyDatumToOb *arg, Datum d);
+static PyObject *PLyBytes_FromBytea(PLyDatumToOb *arg, Datum d);
static PyObject *PLyString_FromDatum(PLyDatumToOb *arg, Datum d);
static PyObject *PLyDict_FromTuple(PLyTypeInfo *, HeapTuple, TupleDesc);
@@ -1723,7 +1769,7 @@ PLy_input_datum_func2(PLyDatumToOb *arg, Oid typeOid, HeapTuple typeTup)
arg->func = PLyLong_FromInt64;
break;
case BYTEAOID:
- arg->func = PLyString_FromBytea;
+ arg->func = PLyBytes_FromBytea;
break;
default:
arg->func = PLyString_FromDatum;
@@ -1814,13 +1860,13 @@ PLyLong_FromInt64(PLyDatumToOb *arg, Datum d)
}
static PyObject *
-PLyString_FromBytea(PLyDatumToOb *arg, Datum d)
+PLyBytes_FromBytea(PLyDatumToOb *arg, Datum d)
{
text *txt = DatumGetByteaP(d);
char *str = VARDATA(txt);
size_t size = VARSIZE(txt) - VARHDRSZ;
- return PyString_FromStringAndSize(str, size);
+ return PyBytes_FromStringAndSize(str, size);
}
static PyObject *
@@ -1917,14 +1963,14 @@ PLyObject_ToBytea(PLyTypeInfo *info,
Assert(plrv != Py_None);
- plrv_so = PyObject_Str(plrv);
+ plrv_so = PyObject_Bytes(plrv);
if (!plrv_so)
- PLy_elog(ERROR, "could not create string representation of Python object");
+ PLy_elog(ERROR, "could not create bytes representation of Python object");
PG_TRY();
{
- char *plrv_sc = PyString_AsString(plrv_so);
- size_t len = PyString_Size(plrv_so);
+ char *plrv_sc = PyBytes_AsString(plrv_so);
+ size_t len = PyBytes_Size(plrv_so);
size_t size = len + VARHDRSZ;
bytea *result = palloc(size);
@@ -1956,22 +2002,30 @@ PLyObject_ToDatum(PLyTypeInfo *info,
PLyObToDatum *arg,
PyObject *plrv)
{
- PyObject *volatile plrv_so = NULL;
+ PyObject *volatile plrv_bo = NULL;
Datum rv;
Assert(plrv != Py_None);
if (PyUnicode_Check(plrv))
- plrv_so = PLyUnicode_Str(plrv);
+ plrv_bo = PLyUnicode_Bytes(plrv);
else
- plrv_so = PyObject_Str(plrv);
- if (!plrv_so)
+ {
+#if PY_MAJOR_VERSION >= 3
+ PyObject *s = PyObject_Str(plrv);
+ plrv_bo = PLyUnicode_Bytes(s);
+ Py_XDECREF(s);
+#else
+ plrv_bo = PyObject_Str(plrv);
+#endif
+ }
+ if (!plrv_bo)
PLy_elog(ERROR, "could not create string representation of Python object");
PG_TRY();
{
- char *plrv_sc = PyString_AsString(plrv_so);
- size_t plen = PyString_Size(plrv_so);
+ char *plrv_sc = PyBytes_AsString(plrv_bo);
+ size_t plen = PyBytes_Size(plrv_bo);
size_t slen = strlen(plrv_sc);
if (slen < plen)
@@ -1984,12 +2038,12 @@ PLyObject_ToDatum(PLyTypeInfo *info,
}
PG_CATCH();
{
- Py_XDECREF(plrv_so);
+ Py_XDECREF(plrv_bo);
PG_RE_THROW();
}
PG_END_TRY();
- Py_XDECREF(plrv_so);
+ Py_XDECREF(plrv_bo);
return rv;
}
@@ -2241,8 +2295,7 @@ static PyMethodDef PLy_plan_methods[] = {
};
static PyTypeObject PLy_PlanType = {
- PyObject_HEAD_INIT(NULL)
- 0, /* ob_size */
+ PyVarObject_HEAD_INIT(NULL, 0)
"PLyPlan", /* tp_name */
sizeof(PLyPlanObject), /* tp_size */
0, /* tp_itemsize */
@@ -2293,8 +2346,7 @@ static PyMethodDef PLy_result_methods[] = {
};
static PyTypeObject PLy_ResultType = {
- PyObject_HEAD_INIT(NULL)
- 0, /* ob_size */
+ PyVarObject_HEAD_INIT(NULL, 0)
"PLyResult", /* tp_name */
sizeof(PLyResultObject), /* tp_size */
0, /* tp_itemsize */
@@ -2353,6 +2405,15 @@ static PyMethodDef PLy_methods[] = {
{NULL, NULL, 0, NULL}
};
+#if PY_MAJOR_VERSION >= 3
+static PyModuleDef PLy_module = {
+ PyModuleDef_HEAD_INIT, /* m_base */
+ "plpy", /* m_name */
+ NULL, /* m_doc */
+ -1, /* m_size */
+ PLy_methods, /* m_methods */
+};
+#endif
/* plan object methods */
static PyObject *
@@ -2940,6 +3001,15 @@ PLy_spi_execute_fetch_result(SPITupleTable *tuptable, int rows, int status)
* language handler and interpreter initialization
*/
+#if PY_MAJOR_VERSION >= 3
+static PyMODINIT_FUNC
+PyInit_plpy(void)
+{
+ return PyModule_Create(&PLy_module);
+}
+#endif
+
+
/*
* _PG_init() - library load-time initialization
*
@@ -2956,7 +3026,13 @@ _PG_init(void)
pg_bindtextdomain(TEXTDOMAIN);
+#if PY_MAJOR_VERSION >= 3
+ PyImport_AppendInittab("plpy", PyInit_plpy);
+#endif
Py_Initialize();
+#if PY_MAJOR_VERSION >= 3
+ PyImport_ImportModule("plpy");
+#endif
PLy_init_interp();
PLy_init_plpy();
if (PyErr_Occurred())
@@ -3002,7 +3078,11 @@ PLy_init_plpy(void)
if (PyType_Ready(&PLy_ResultType) < 0)
elog(ERROR, "could not initialize PLy_ResultType");
+#if PY_MAJOR_VERSION >= 3
+ plpy = PyModule_Create(&PLy_module);
+#else
plpy = Py_InitModule("plpy", PLy_methods);
+#endif
plpy_dict = PyModule_GetDict(plpy);
/* PyDict_SetItemString(plpy, "PlanType", (PyObject *) &PLy_PlanType); */
@@ -3348,12 +3428,29 @@ PLy_free(void *ptr)
}
/*
- * Convert a Python unicode object to a Python string object in
+ * Convert a Unicode object to a Python string.
+ */
+static PyObject*
+PLyUnicode_Str(PyObject *unicode)
+{
+#if PY_MAJOR_VERSION >= 3
+ /* In Python 3, this is a noop. */
+ Py_INCREF(unicode);
+ return unicode;
+#else
+ /* In Python 2, this means converting the Unicode to bytes in the
+ * server encoding. */
+ return PLyUnicode_Bytes(unicode);
+#endif
+}
+
+/*
+ * Convert a Python unicode object to a Python string/bytes object in
* PostgreSQL server encoding. Reference ownership is passed to the
* caller.
*/
static PyObject*
-PLyUnicode_Str(PyObject *unicode)
+PLyUnicode_Bytes(PyObject *unicode)
{
PyObject *rv;
const char *serverenc;
@@ -3375,13 +3472,44 @@ PLyUnicode_Str(PyObject *unicode)
/*
* Convert a Python unicode object to a C string in PostgreSQL server
* encoding. No Python object reference is passed out of this
- * function.
+ * function. The result is palloc'ed.
+ *
+ * Note that this function is disguised as PyString_AsString() when
+ * using Python 3. That function retuns a pointer into the internal
+ * memory of the argument, which isn't exactly the interface of this
+ * function. But in either case you get a rather short-lived
+ * reference that you ought to better leave alone.
*/
static char *
PLyUnicode_AsString(PyObject *unicode)
{
- PyObject *o = PLyUnicode_Str(unicode);
- char *rv = PyString_AsString(o);
+ PyObject *o = PLyUnicode_Bytes(unicode);
+ char *rv = pstrdup(PyBytes_AsString(o));
Py_XDECREF(o);
return rv;
}
+
+#if PY_MAJOR_VERSION >= 3
+/*
+ * Convert a C string in the PostgreSQL server encoding to a Python
+ * unicode object. Reference ownership is passed to the caller.
+ */
+static PyObject *
+PLyUnicode_FromString(const char *s)
+{
+ char *utf8string;
+ PyObject *o;
+
+ utf8string = (char *) pg_do_encoding_conversion((unsigned char *) s,
+ strlen(s),
+ GetDatabaseEncoding(),
+ PG_UTF8);
+
+ o = PyUnicode_FromString(utf8string);
+
+ if (utf8string != s)
+ pfree(utf8string);
+
+ return o;
+}
+#endif /* PY_MAJOR_VERSION >= 3 */
Peter Eisentraut <peter_e@gmx.net> writes:
Here's the patch to support Python >=3.1 with PL/Python. The
compatibility code is mostly in line with the usual 2->3 C porting
practice and is documented inline.
There was considerable debate earlier about whether we wanted to treat
Python 3 as a separate PL so it could be available in parallel with
plpython 2, because of the user-level coding incompatibilities. It
looks like this patch simply ignores that problem. What is going to
happen to plpython functions that depend on 2.x behavior?
regards, tom lane
On Nov 12, 2009, at 12:54 PM, Peter Eisentraut wrote:
Here's the patch to support Python >=3.1 with PL/Python.
:\
I was hoping to be able to use Python 3 to draw a clear distinction between plpython and the would-be "plpython3" that I've been working on. I understand that you're not in favor of a brand new implementation for Python 3. Despite my dislike for that position(well, it would seem to be in opposition to my initiative, so naturally =), I don't entirely disagree with your rationale[wrt doing things more incrementally]. For me, plpython has never been what I would call a pleasure to use, and many of the gripes that I have with it are, IMO, entrenched far enough into the implementation that any efforts to change it would(should? =) cause unacceptable breakage in user applications(?). Well, as far as additional Python interfaces are concerned, a lot of redundant functionality, but that's not even the half of it.
[I was hoping to get to a status message this weekend,
but it seems like I should follow-up here. =]
So here's where I'm at:
--
Mostly documentation improvements since I last pinged -hackers.
Still, *sigh*, filling in documentation and fighting bugs as I go.
Currently resolving a bug instantiating MD arrays from nested lists.
Once I'm "finished" with the docs, I'm going to start looking for refcount leaks.
No major additions or changes are planned, but I have been making some minor additions as I write more docs.
Overview/Features:
http://wiki.postgresql.org/wiki/WIP:plpython3
Documentation:
http://python.projects.postgresql.org/pldocs/plpython3.html
git repo[see the plpython3 branch]:
http://git.postgresql.org/gitweb?p=plpython3.git;a=summary
Most of the documented interfaces have tests. I only have two platforms at my disposal, so I do fear that this will not "just work" on all of PG's supported platforms. Specifically, I've ran the tests on freebsd/amd64 and Mac10.6/intel(of course 10.5 as well for some earlier revisions). [err, actually, it's been a while since I ran the tests on freebsd.]
--
plpython3 is turning out to be kinda beefy(~974K diff[eh, there is some fluff in there]), and I can't say that I've seen much interest in it, so I can't really blame anyone if -hackers ends up taking a pass on it. (python3 is too far away for most folk to care? folk are content with plpython?)
eh, cheers, either way. =)
On tor, 2009-11-12 at 16:06 -0500, Tom Lane wrote:
Peter Eisentraut <peter_e@gmx.net> writes:
Here's the patch to support Python >=3.1 with PL/Python. The
compatibility code is mostly in line with the usual 2->3 C porting
practice and is documented inline.There was considerable debate earlier about whether we wanted to treat
Python 3 as a separate PL so it could be available in parallel with
plpython 2, because of the user-level coding incompatibilities. It
looks like this patch simply ignores that problem.
Exactly how to package that is something to be determined by the
packagers, and we can give them the support they need. But first you
need code that works with Python 3, which is what this patch does.
What is going to happen to plpython functions that depend on 2.x behavior?
The porting path from 2 to 3 is pretty well established. You first port
to 2.6, then remove all the old features, then move to 3.x. This is not
something we have to reinvent. The only significant differences that
you can't use in 2.6 without future imports are unicode literals and the
print function, both of which are not in common use in PL/Python.
On tor, 2009-11-12 at 18:42 -0700, James Pye wrote:
For me, plpython has never been what I would call a pleasure to use,
and many of the gripes that I have with it are, IMO, entrenched far
enough into the implementation that any efforts to change it
would(should? =) cause unacceptable breakage in user applications(?).
Has this list of gripes ever been brought up and discussed in this
forum?
On Nov 13, 2009, at 4:47 AM, Peter Eisentraut wrote:
Has this list of gripes ever been brought up and discussed in this
forum?
Some are TODOs, so in part by other people. Some were briefly touched on in the recent past discussions(around the time that I announced the WIP). Native typing vs conversion, function fragments vs function modules.
I don't think native typing has seen any actual discussion, but I do recall mentioning it as something that I wanted to do(implicitly griped?).
...
There is a difference in the situation from the discussion before. Prior, it was, "I would like to implement a new PL for Python 3 with these features", and now, it is, "I have implemented a new PL for Python 3 with these features".
Simply, -hackers can choose among moving forward with Python 3 support in plpython or going with "plpython3" or even both, I suppose(?). Naturally, I'm biased toward something that involves plpython3, so I don't think I can(should?) be of much help to -hackers as a Python & PG user in any such decision. Of course, excepting the provision of justifications for my implementation/design choices...
I would really love to see some input from Python users.
I certainly don't want to waste time trying to get something into pgsql that Python users don't want.
[here's a gripe that I haven't brought up as I think it is a matter of taste]
I find (plpython) trigger functions to be a bit odd. I think it's the way in which manipulation/suppression decisions are made in BEFORE ROW triggers(return "OK", "SKIP", etc).. [label this as opinion at this point as I have yet to be able to nail down what, specifically, is "wrong" or un-pythonic about them.]
Also, having distinct entry points to handle trigger events helps reduce potential errors by forcing the user to explicitly state the events that the trigger function can handle. Currently, in plpython, users *should* do sanity checks.
Function modules opened the door for implementing this in a natural way, multiple functions(entry points) in the function module.
http://python.projects.postgresql.org/pldocs/plpython3-programming.html#PLPYTHON3-FUNCTIONS-TRIGGERS
On fre, 2009-11-13 at 11:27 -0700, James Pye wrote:
Some are TODOs, so in part by other people. Some were briefly touched
on in the recent past discussions(around the time that I announced the
WIP). Native typing vs conversion, function fragments vs function
modules.
I'm of course only one user, but these two features don't excite me at
all, and certainly not enough to go through the pain of dealing with a
second implementation.
On Nov 15, 2009, at 6:37 AM, Peter Eisentraut wrote:
but these two features don't excite me at all,
hrm.. at all?
I can see how function modules might look like a half-step backwards from function fragments at first, but the benefits of a *natural* initialization section (the module body) was enough to convince me. The added value on the PL developer's side was also compelling. Tracebacks were trivial to implement, and there is no need to munge the function's source. It seemed like a win all around...
AFA native typing is concerned, I think the flexibility and potential it offers is useful, no? Already, plpython3 provides properties on PG's datetime types to access the date_part()'s of the object.
OTOH, for folk who primarily use the PL to access functionality in Python modules(bindings), native typing may be of no direct utility as they will likely need to convert anyways. (If that's your common use-case, then the absence of interest in native typing is quite understandable.)
[looking at the PL/Python todo list..]
Excepting DB-API and trusted, I believe all the current PL/Python TODOs are fulfilled or N/A in plpython3... ugh, the docs are not yet complete, but I like to think of them as "better" anyways. :P
the pain of dealing with a second implementation.
What pain are you anticipating? Maintenance?
On sön, 2009-11-15 at 18:39 -0700, James Pye wrote:
I can see how function modules might look like a half-step backwards from function fragments at first, but the benefits of a *natural* initialization section (the module body) was enough to convince me. The added value on the PL developer's side was also compelling. Tracebacks were trivial to implement, and there is no need to munge the function's source. It seemed like a win all around...
The question is whether it helps the user, not the implementer. As far
as I can tell, it just creates more typing for no benefit whatsoever.
Also, it's inconsistent with normal Python script files and with other
PLs.
AFA native typing is concerned, I think the flexibility and potential it offers is useful, no? Already, plpython3 provides properties on PG's datetime types to access the date_part()'s of the object.
OTOH, for folk who primarily use the PL to access functionality in Python modules(bindings), native typing may be of no direct utility as they will likely need to convert anyways. (If that's your common use-case, then the absence of interest in native typing is quite understandable.)
Right, if I use PL/Python, I do it because I want to use Python. I
don't need another PostgreSQL implementation on top of Python. The
maintenance effort required to keep those two consistent aside.
Again, I'm only one user. But so far I haven't seen anyone else speak
up here, and clearly accepting this for inclusion will need nontrivial
convincing.
the pain of dealing with a second implementation.
What pain are you anticipating? Maintenance?
Right.
Again, I'm only one user. But so far I haven't seen anyone else speak
up here, and clearly accepting this for inclusion will need nontrivial
convincing.
Well, FWIW, I am excited about better type integration.
Also, I am a little skeptical about this patch. I am sorry if this has
already been discussed, but would this mean that I need to choose
whether pl/python is built against Python 2.* or Python 3.*?
-Nathan
Nathan Boley <npboley@gmail.com> writes:
Also, I am a little skeptical about this patch. I am sorry if this has
already been discussed, but would this mean that I need to choose
whether pl/python is built against Python 2.* or Python 3.*?
Yes. That's exactly what I was complaining about upthread. I'm not
a Python user, but from what I can gather of the 2-to-3 changes,
having to choose one at package build time is going to be a disaster.
regards, tom lane
On Wed, 2009-11-18 at 12:06 -0500, Tom Lane wrote:
Nathan Boley <npboley@gmail.com> writes:
Also, I am a little skeptical about this patch. I am sorry if this has
already been discussed, but would this mean that I need to choose
whether pl/python is built against Python 2.* or Python 3.*?Yes. That's exactly what I was complaining about upthread. I'm not
a Python user, but from what I can gather of the 2-to-3 changes,
having to choose one at package build time is going to be a disaster.
Agreed. We really need to have a plpython and plpython3. Heck this would
play nicely too because we support backward compatibility but also
upward version differences.
Joshua D. Drake
regards, tom lane
--
PostgreSQL.org Major Contributor
Command Prompt, Inc: http://www.commandprompt.com/ - 503.667.4564
Consulting, Training, Support, Custom Development, Engineering
If the world pushes look it in the eye and GRR. Then push back harder. - Salamander
"Joshua D. Drake" <jd@commandprompt.com> writes:
On Wed, 2009-11-18 at 12:06 -0500, Tom Lane wrote:
Yes. That's exactly what I was complaining about upthread. I'm not
a Python user, but from what I can gather of the 2-to-3 changes,
having to choose one at package build time is going to be a disaster.
Agreed. We really need to have a plpython and plpython3.
Peter was concerned about duplicative maintenance effort, but what I
think this patch shows is that (at least for the near future) both
could be built from a single source file. What we need is configure
and makefile support to do that.
regards, tom lane
On Wed, 2009-11-18 at 12:28 -0500, Tom Lane wrote:
"Joshua D. Drake" <jd@commandprompt.com> writes:
On Wed, 2009-11-18 at 12:06 -0500, Tom Lane wrote:
Yes. That's exactly what I was complaining about upthread. I'm not
a Python user, but from what I can gather of the 2-to-3 changes,
having to choose one at package build time is going to be a disaster.Agreed. We really need to have a plpython and plpython3.
Peter was concerned about duplicative maintenance effort, but what I
think this patch shows is that (at least for the near future) both
could be built from a single source file. What we need is configure
and makefile support to do that.
Ahh, so we would have:
--enable-plpython2=/usr/bin/python2
--enable-plpython3=/usr/bin/python3
?
That seems reasonable if we can run both. Although I wonder if longer
term (2.x is going to be support a long time) we will end up with
frustration within the single source file trying to keep things
straight.
Joshua D. Drake
regards, tom lane
--
PostgreSQL.org Major Contributor
Command Prompt, Inc: http://www.commandprompt.com/ - 503.667.4564
Consulting, Training, Support, Custom Development, Engineering
If the world pushes look it in the eye and GRR. Then push back harder. - Salamander
"Joshua D. Drake" <jd@commandprompt.com> writes:
On Wed, 2009-11-18 at 12:28 -0500, Tom Lane wrote:
Peter was concerned about duplicative maintenance effort, but what I
think this patch shows is that (at least for the near future) both
could be built from a single source file.
That seems reasonable if we can run both. Although I wonder if longer
term (2.x is going to be support a long time) we will end up with
frustration within the single source file trying to keep things
straight.
Once it gets to the point where it's more trouble to keep them together
than not, we can split the source. But judging from this patch, a
single source file is the ticket for the moment.
regards, tom lane
Here's the patch to support Python >=3.1 with PL/Python. The
compatibility code is mostly in line with the usual 2->3 C porting
practice and is documented inline.
I took a cursory look at this patch and, while the logic seems sound
and roughly in line with the suggested python porting procedure, I'm
not quite certain what this implies for potential future patches.
For instance, if I wanted to write a type converter for bytea -> the
python 3 byte type would the expectation be that I ensure that it
works in Python 2? Or is an ifdef that ignores it in the case of
Python 2 OK, and we can just put a note in the docs.
Also, how far back do we want to maintain 2.x compatibility? 2.0? If I
wanted to submit a patch that makes use of the list sort method, do I
need to ensure that it can either use the cmp arguments or a key
argument?
What if I wanted to implement a set returning function that made use
of an iterators next() method. Would I just put ifdefs around the code
or a preprocessor definition that defines NEXT as next() for Python
2.x and __next__() for 3.x?
I guess that my first impression is that Python broke compatibility
for a reason, and that either plpython can't evolve, or it will
quickly become impossible to maintain. That being said, I mostly buy
the maintenance arguments from the previous discussion, but if we want
to have plpython and plpython3, a bunch of defines and ifdefs does not
seem like the best way to do this.
Would a better approach be to maintain compatibility layer? ie
plython_compat.h/c
plython2.c
plython3.c
Then patches that apply to a python3 can be applied to plython3.c and
any changed function can be ripped out of plython_compat and moved
into plpython2.
I'm sorry to snipe from the sidelines like this. If we didn't expect
plpython to evolve then this patch seems like the correct approach,
but there is clearly some desire to expand plpython and following this
path seems like it will end in a much more painful split in the future
or a necessary rewrite.
If there is some consensus that this is the best approach, then I will
do a more comprehensive review.
-Nathan
On Nov 18, 2009, at 8:37 AM, Peter Eisentraut wrote:
The question is whether it helps the user, not the implementer.
Sure, but do you have a patch waiting to implement tracebacks?
I'd argue the reason it's never been done is due to the way procedures are currently managed in PL/Python. And *without some significant refactoring*, any patch fully implementing tracebacks is going to be a seriously ugly hack.
What helped the implementer here would help the user.
As far
as I can tell, it just creates more typing for no benefit whatsoever.
"def main(*args):" is annoying, but not entirely lamentable...
It's explicit, as well(no need to document munging that occurs behind the scenes).
Also, compare the cases where you need to cache some initialized data:
if 'key' not in SD:
...
SD['key'] = my_newly_initialized_data
...
With function modules, SD is not needed as you have your module globals to keep your locally cached data in:
...
data = my_newly_initialized_data
def main(*args):
...
Also, it's inconsistent with normal Python script files
Hinges on whether "normal" is actually normal.
I often use the __name__ convention in script files myself:
if __name__ == '__main__':
main(...)
That is, using that convention, the script can be import'd and used without executing the "script functionality". (It has proven to be very handy a few times now)
and with other PLs.
I don't understand why that's a significant enough interest to note.
I don't need another PostgreSQL implementation on top of Python.
Indeed, and I do understand that. That is, I have removed some features with that very thought in mind. (OTOH, I consider the date_part properties on datetime types to be special: too likely useful.)
[tho, "PostgreSQL implementation"? I think I understand what you were getting at, but..]
The maintenance effort required to keep those two consistent aside.
I don't think there are many consistency issues here.
What did you have in mind?
Again, I'm only one user. But so far I haven't seen anyone else speak up here, and clearly accepting this for inclusion will need nontrivial convincing.
Agreed. It would seem quite doomed.
At this point, I'm not going to try getting it into PG. (apparent futility and such)
On Nov 18, 2009, at 1:36 PM, James Pye wrote:
At this point, I'm not going to try getting it into PG. (apparent futility and such)
ugh, on second thought, I think I've written a bit too much code to stop now. I'm going to get plpython3 as far as I can and submit it to the next commitfest.
On ons, 2009-11-18 at 09:48 -0800, Joshua D. Drake wrote:
Although I wonder if longer
term (2.x is going to be support a long time) we will end up with
frustration within the single source file trying to keep things
straight.
There are five million Python modules with C code out there with the
same problem. Considerable effort has been put in by Python upstream to
make the effort manageable. No one in their right mind is going to
create two separate source files just because in the future the mythical
differences will be too big, when clearly the effort is going into a
direction to reduce the differences.
If you look into the source file, there is already special code for
Python 2.2, 2.3, 2.4, 2.5, 2.6, and now 3.1. The chunk for 3.1 is a bit
bigger, but only a bit, and well, that's why it's 3.x and not 2.x. No
one has ever suggested, we might need to support Python 2.2 for a long
time, let's create a separate source file.
I agree, there will probably need to be some configuration/build support
on top of this, but that's something we should work out independently of
how to manage the source file.
On ons, 2009-11-18 at 08:43 -0800, Nathan Boley wrote:
Again, I'm only one user. But so far I haven't seen anyone else speak
up here, and clearly accepting this for inclusion will need nontrivial
convincing.Well, FWIW, I am excited about better type integration.
Let's clarify, as there are two different models being proposed here.
The first approach, which is currently implemented (and some patches
pending), is to convert a PostgreSQL type to the "nearest" Python type.
For example, text to string, int to int, array to list, timestamp to
datetime.datetime, etc. The other approach, which is what James Pye's
new implementation proposes (as I understand it), is to convert
PostgreSQL types into specially made Python objects, such as
Postgres.types.record or Postgres.types.timestamp.
Also, I am a little skeptical about this patch. I am sorry if this has
already been discussed, but would this mean that I need to choose
whether pl/python is built against Python 2.* or Python 3.*?
Yeah, see later discussion about how to resolve this. But I think in
practice, unless you use lots of print statements in your stored
procedures (?!?), this problem is exaggerated.
On ons, 2009-11-18 at 11:32 -0800, Nathan Boley wrote:
I took a cursory look at this patch and, while the logic seems sound
and roughly in line with the suggested python porting procedure, I'm
not quite certain what this implies for potential future patches.For instance, if I wanted to write a type converter for bytea -> the
python 3 byte type would the expectation be that I ensure that it
works in Python 2? Or is an ifdef that ignores it in the case of
Python 2 OK, and we can just put a note in the docs.
Note that this is already implemented. The main point of the patch is
to provide a small compatibility layer so that these kinds of issues are
practically nonexistent. The fact that you didn't notice might prove
that the patch does its job. ;-)
Also, how far back do we want to maintain 2.x compatibility? 2.0?
We handle this on an ad hoc basis. We currently support Python 2.2 and
later, and this cutoff exists -- this is my interpretation of history --
because 2.2 introduced iterators and no one bothered(?) to put ifdefs
around the code in PL/Python that provides iterator support. Over the
years, we will probably drop support for other older Python versions,
but there is no process or plan for that. Right now, the support for
Python 2.2 is about three lines, so it's not a bother, but when someone
comes and implements a major feature that, say, requires Python 2.3, we
can probably drop 2.2. But when the major feature requires 2.6, we
probably don't want to drop 2.5 quite yet at this time. It's a judgment
call.
If I
wanted to submit a patch that makes use of the list sort method, do I
need to ensure that it can either use the cmp arguments or a key
argument?
Any patch one is likely to submit will be a C patch, not a Python patch.
But anyway, the "key" argument was introduced in Python 2.4, and so we'd
have to come to a decision in the community about whether Python 2.3
support is worth keeping versus the value of that new feature. See above.
But anyway, this problem has nothing to do with my patch; it has already
existed in the same form forever.
What if I wanted to implement a set returning function that made use
of an iterators next() method. Would I just put ifdefs around the code
or a preprocessor definition that defines NEXT as next() for Python
2.x and __next__() for 3.x?
Again, you would likely submit a C patch, and the iterator API is the
same between 2.x and 3.x.
I guess that my first impression is that Python broke compatibility
for a reason, and that either plpython can't evolve, or it will
quickly become impossible to maintain.
I think this is an exaggeration of reality. Python 3 removed deprecated
features. There is a perfectly good migration path that covers most
code: Switch to Python 2.6, switch to the new features, remove the old
features, switch to Python 3.x. This applies both on the Python and the
C level. They did not break compatibility with the intention of making
every module author out there reimplement their thing from scratch.
Otherwise Python 2.6 would make very little sense at all.
Take a look at an example closer to home: PostgreSQL breaks C API
compatibility in almost every major release. We do this to remove cruft
and support new features. The intent is not to make Slony and PostGIS
and all the other modules reimplement their product from scratch every
time. They put in a few ifdefs, sometimes they complain about it ;-),
and then the problem is solved.
That being said, I mostly buy
the maintenance arguments from the previous discussion, but if we want
to have plpython and plpython3, a bunch of defines and ifdefs does not
seem like the best way to do this.
These ifdefs were not my idea. They are in some cases directly and in
some cases in spirit from the Python 2.6 header files, so they are the
official way to do this.
Would a better approach be to maintain compatibility layer? ie
plython_compat.h/c
plython2.c
plython3.cThen patches that apply to a python3 can be applied to plython3.c and
any changed function can be ripped out of plython_compat and moved
into plpython2.
As I tried to explain above, we have always had a rolling feature model
of sorts, even across various Python 2.x versions. If you want to try
it out, you could take the current source and split it up into
plpython22.c, plpython23.c, etc. and see if that becomes useful.
On Nov 19, 2009, at 3:12 AM, Peter Eisentraut wrote:
The other approach, which is what James Pye's
new implementation proposes (as I understand it), is to convert
PostgreSQL types into specially made Python objects, such as
Postgres.types.record or Postgres.types.timestamp.
Convert is not a good word choice. The Datum of the parameter is stored inside a new Python object(that only holds a Datum). So more like "copied into Python memory", and associated with its respective type. Wrapped in a Python object?
One cool thing about doing it this way, is that if you just pass parameters forward to a prepared statement, there's no type I/O overhead. Not a huge performance win for common cases, but if someone were passing larger arrays around, it could be quite beneficial.
On ons, 2009-11-18 at 12:28 -0500, Tom Lane wrote:
"Joshua D. Drake" <jd@commandprompt.com> writes:
On Wed, 2009-11-18 at 12:06 -0500, Tom Lane wrote:
Yes. That's exactly what I was complaining about upthread. I'm not
a Python user, but from what I can gather of the 2-to-3 changes,
having to choose one at package build time is going to be a disaster.Agreed. We really need to have a plpython and plpython3.
Peter was concerned about duplicative maintenance effort, but what I
think this patch shows is that (at least for the near future) both
could be built from a single source file. What we need is configure
and makefile support to do that.
By the way, it occurred to me that having two different versions of
libpython loaded into the same process is probably not going to work
sanely. So whatever solution we come up with for the Python 3
transition, the possibilities for a jolly back-and-forth are probably
going to be quite limited.
On ons, 2009-11-18 at 13:36 -0700, James Pye wrote:
On Nov 18, 2009, at 8:37 AM, Peter Eisentraut wrote:
The question is whether it helps the user, not the implementer.
Sure, but do you have a patch waiting to implement tracebacks?
I'd argue the reason it's never been done is due to the way procedures are currently managed in PL/Python. And *without some significant refactoring*, any patch fully implementing tracebacks is going to be a seriously ugly hack.
What helped the implementer here would help the user.
But you wouldn't, for example, get away with breaking SQL (or even
improving it incompatibly) to facilitate a better elog.
As far
as I can tell, it just creates more typing for no benefit whatsoever."def main(*args):" is annoying, but not entirely lamentable...
It's explicit, as well(no need to document munging that occurs behind the scenes).Also, compare the cases where you need to cache some initialized data:
if 'key' not in SD:
...
SD['key'] = my_newly_initialized_data
...With function modules, SD is not needed as you have your module globals to keep your locally cached data in:
...
data = my_newly_initialized_datadef main(*args):
...
I can see that this creates other options for structuring code, but it
doesn't actually match my way of thinking. (Obviously, I'm biased, but
anyway.) I think of a PL/Python function as a Python script file stored
in the database. When you call it, arguments are passed just like a
Python script receives arguments from the shell. When Python scripts
want to share data, they might use a file (or perhaps a database server
in advanced cases) and do
if not file exists:
create the file
fill it with data
This is in my mind quite analogous to how the SD business works.
The analogy to your approach, as I understand it, would be that multiple
instances of the same script file will automatically share their global
variables. That could be quite interesting, actually, but it's not how
it works, and in most cases it's better that way.
Peter Eisentraut <peter_e@gmx.net> writes:
By the way, it occurred to me that having two different versions of
libpython loaded into the same process is probably not going to work
sanely.
Why not? There's no way they'd even know about each other. We tell
the loader not to make the symbols globally visible.
But in any case, my main concern here is that I don't want to have
to predetermine which python version a user of Red Hat/Fedora will
have to use. If they can only use one at a time, that's still a
good bit better than not having a choice at all.
regards, tom lane
On Nov 19, 2009, at 11:32 AM, Peter Eisentraut wrote:
But you wouldn't, for example, get away with breaking SQL (or even
improving it incompatibly) to facilitate a better elog.
This doesn't fit the situation.
I'm not breaking PL/Python. I'm trying to add PL/Python3. =)
I think of a PL/Python function as a Python script file stored
in the database.
For Python, I think that's a mistake. Python scripts are independent applications.
[tho, I think this does illuminate our perspectives...]
On tor, 2009-11-19 at 13:43 -0500, Tom Lane wrote:
But in any case, my main concern here is that I don't want to have
to predetermine which python version a user of Red Hat/Fedora will
have to use. If they can only use one at a time, that's still a
good bit better than not having a choice at all.
By the way, mod_wsgi supports Python 3 already (same patch as here, in
principle). From the Fedora wiki page, I gather that no one has really
looked into packaging that yet for Python 3, but if someone does, maybe
we can cross-inspire ourselves.
On tor, 2009-11-19 at 13:12 -0700, James Pye wrote:
I think of a PL/Python function as a Python script file stored
in the database.For Python, I think that's a mistake. Python scripts are independent applications.
Is there any precedent for the sort of behavior that you are
implementing, that is, automatic sharing of variables between
independent executions of the same source container?
On Nov 20, 2009, at 12:02 AM, Peter Eisentraut wrote:
Is there any precedent for the sort of behavior that you are
implementing, that is, automatic sharing of variables between
independent executions of the same source container?
import foo
# bar is a regular, def'd function.
foo.bar()
...
# even in another thread, doesn't matter..
foo.bar()
In either call, foo.bar()'s globals() is the same dictionary object(the foo module's dictionary).
A plpython3 function *is* a Python module.
On fre, 2009-11-20 at 01:20 -0700, James Pye wrote:
On Nov 20, 2009, at 12:02 AM, Peter Eisentraut wrote:
Is there any precedent for the sort of behavior that you are
implementing, that is, automatic sharing of variables between
independent executions of the same source container?import foo
# bar is a regular, def'd function.
foo.bar()...
# even in another thread, doesn't matter..
foo.bar()In either call, foo.bar()'s globals() is the same dictionary object(the foo module's dictionary).
That's not what I meant, because this is the same execution of the same
source container, with threads explicitly started somewhere. You could
do the same in a plpython function (in theory, at least).
What I mean is more like, you execute the same source file twice in a
row, and the global variables are saved for the second run.
On Nov 20, 2009, at 1:26 AM, Peter Eisentraut wrote:
because this is the same execution
Hrm, not necessarily. foo could be imported by another, completely independent part of the program. foo is cached in sys.modules. bar() is executed and it's still the same globals(). shared.
Am 19.11.2009 18:01, schrieb James Pye:
On Nov 19, 2009, at 3:12 AM, Peter Eisentraut wrote:
The other approach, which is what James Pye's
new implementation proposes (as I understand it), is to convert
PostgreSQL types into specially made Python objects, such as
Postgres.types.record or Postgres.types.timestamp.Convert is not a good word choice. The Datum of the parameter is stored inside a new Python object(that only holds a Datum). So more like "copied into Python memory", and associated with its respective type. Wrapped in a Python object?
Yes "wrapped" is the term commonly used for that. And I must say I like
it and I used plpy where I could.
One cool thing about doing it this way, is that if you just pass parameters forward to a prepared statement, there's no type I/O overhead. Not a huge performance win for common cases, but if someone were passing larger arrays around, it could be quite beneficial.
Exactly and you have all the meta information about the original
postgres type. IIRC there were some thoughts of having something like
that in a DBAPI interface as well (similar for example to cx_Oracle).
Tino
Attachments:
On tor, 2009-11-12 at 16:06 -0500, Tom Lane wrote:
There was considerable debate earlier about whether we wanted to treat
Python 3 as a separate PL so it could be available in parallel with
plpython 2, because of the user-level coding incompatibilities. It
looks like this patch simply ignores that problem. What is going to
happen to plpython functions that depend on 2.x behavior?
I have a proposal for how to handle this, and a prototype patch
attached. This follows essentially what the CPython distribution itself
does, which will make this tolerably easy to follow for users.
We install plpython as plpython2.so or plpython3.so, depending on the
version used to build it. Then, plpython.so is a symlink to
plpython2.so.
We then create three language definition templates:
plpythonu -> plpython.so
plpython2u -> plpython2.so
plpython3u -> plpython3.so
In the far future we flip the default symlink to plpython3.so, maybe in
about 5 years when Python 2.x expires.
This gives the users the following options and scenarios:
- Existing users don't have to do anything, until maybe in five years
they will notice that their OS doesn't ship Python 2 anymore and they
will have to act anyway. In practice, by then they might have adjusted
their coding style to Python 2.6/2.7 norms and their functions will
migrate to 3.x without change.
- Users who know that they have heavily Python 2.x dependent code and
don't want to ever change it can make use of the plpython2u language
name, just like they should probably change their scripts to use
something like #!/usr/bin/python2.
- Users who want to dive into Python 3 can use the plpython3u language
name, which will basically keep working forever by today's standards.
Those users would probably also use #!/usr/bin/python3 or the like in
their scripts. In the far future they might like to remove the "3".
- Daredevils can change symlink manually and make plpython3.so the
default plpythonu implementation. Those people would probably also
make /usr/bin/python be version 3.
Comments?
Attachments:
python3-build.patchtext/x-patch; charset=UTF-8; name=python3-build.patchDownload
diff --git a/config/python.m4 b/config/python.m4
index 9160a2b..32fff43 100644
--- a/config/python.m4
+++ b/config/python.m4
@@ -30,10 +30,12 @@ else
AC_MSG_ERROR([distutils module not found])
fi
AC_MSG_CHECKING([Python configuration directory])
+python_majorversion=`${PYTHON} -c "import sys; print(sys.version[[0]])"`
python_version=`${PYTHON} -c "import sys; print(sys.version[[:3]])"`
python_configdir=`${PYTHON} -c "from distutils.sysconfig import get_python_lib as f; import os; print(os.path.join(f(plat_specific=1,standard_lib=1),'config'))"`
python_includespec=`${PYTHON} -c "import distutils.sysconfig; print('-I'+distutils.sysconfig.get_python_inc())"`
+AC_SUBST(python_majorversion)[]dnl
AC_SUBST(python_version)[]dnl
AC_SUBST(python_configdir)[]dnl
AC_SUBST(python_includespec)[]dnl
diff --git a/src/Makefile.global.in b/src/Makefile.global.in
index ca7f499..7a6e3a9 100644
--- a/src/Makefile.global.in
+++ b/src/Makefile.global.in
@@ -171,6 +171,7 @@ python_libdir = @python_libdir@
python_libspec = @python_libspec@
python_additional_libs = @python_additional_libs@
python_configdir = @python_configdir@
+python_majorversion = @python_majorversion@
python_version = @python_version@
krb_srvtab = @krb_srvtab@
diff --git a/src/include/catalog/pg_pltemplate.h b/src/include/catalog/pg_pltemplate.h
index 8cdedb4..cbb0a33 100644
--- a/src/include/catalog/pg_pltemplate.h
+++ b/src/include/catalog/pg_pltemplate.h
@@ -73,5 +73,7 @@ DATA(insert ( "pltclu" f f "pltclu_call_handler" _null_ _null_ "$libdir/pltcl"
DATA(insert ( "plperl" t t "plperl_call_handler" "plperl_inline_handler" "plperl_validator" "$libdir/plperl" _null_ ));
DATA(insert ( "plperlu" f f "plperl_call_handler" "plperl_inline_handler" "plperl_validator" "$libdir/plperl" _null_ ));
DATA(insert ( "plpythonu" f f "plpython_call_handler" _null_ _null_ "$libdir/plpython" _null_ ));
+DATA(insert ( "plpython2u" f f "plpython_call_handler" _null_ _null_ "$libdir/plpython2" _null_ ));
+DATA(insert ( "plpython3u" f f "plpython_call_handler" _null_ _null_ "$libdir/plpython3" _null_ ));
#endif /* PG_PLTEMPLATE_H */
diff --git a/src/pl/plpython/Makefile b/src/pl/plpython/Makefile
index 840b874..d11525d 100644
--- a/src/pl/plpython/Makefile
+++ b/src/pl/plpython/Makefile
@@ -36,7 +36,7 @@ override CPPFLAGS := -I$(srcdir) $(python_includespec) $(CPPFLAGS)
rpathdir = $(python_libdir)
-NAME = plpython
+NAME = plpython$(python_majorversion)
OBJS = plpython.o
@@ -56,7 +56,10 @@ endif
SHLIB_LINK = $(python_libspec) $(python_additional_libs) $(filter -lintl,$(LIBS))
-REGRESS_OPTS = --dbname=$(PL_TESTDB) --load-language=plpythonu
+REGRESS_OPTS = --dbname=$(PL_TESTDB)
+ifeq ($(python_majorversion),2)
+REGRESS_OPTS += --load-language=plpythonu
+endif
REGRESS = \
plpython_schema \
plpython_populate \
@@ -83,10 +86,16 @@ include $(top_srcdir)/src/Makefile.shlib
all: all-lib
install: all installdirs install-lib
+ifeq ($(python_majorversion),2)
+ cd '$(DESTDIR)$(pkglibdir)' && rm -f plpython$(DLSUFFIX) && $(LN_S) $(shlib) plpython$(DLSUFFIX)
+endif
installdirs: installdirs-lib
uninstall: uninstall-lib
+ifeq ($(python_majorversion),2)
+ rm -f '$(DESTDIR)$(pkglibdir)/plpython$(DLSUFFIX)'
+endif
ifneq (,$(findstring 3.,$(python_version)))
# Adjust regression tests for Python 3 compatibility
@@ -100,6 +109,8 @@ prep3:
-e 's/\bu"/"/g' \
-e "s/\bu'/'/g" \
-e "s/def next\b/def __next__/g" \
+ -e "s/LANGUAGE plpythonu\b/LANGUAGE plpython3u/gi" \
+ -e "s/LANGUAGE plpython2u\b/LANGUAGE plpython3u/gi" \
$$file >`echo $$file | sed 's,$(srcdir),3,'`; \
done
diff --git a/src/pl/plpython/expected/plpython_test.out b/src/pl/plpython/expected/plpython_test.out
index c5cfe5a..a229b18 100644
--- a/src/pl/plpython/expected/plpython_test.out
+++ b/src/pl/plpython/expected/plpython_test.out
@@ -1,4 +1,5 @@
-- first some tests of basic functionality
+CREATE LANGUAGE plpython2u;
-- really stupid function just to get the module loaded
CREATE FUNCTION stupid() RETURNS text AS 'return "zarkon"' LANGUAGE plpythonu;
select stupid();
@@ -7,6 +8,14 @@ select stupid();
zarkon
(1 row)
+-- check 2/3 versioning
+CREATE FUNCTION stupidn() RETURNS text AS 'return "zarkon"' LANGUAGE plpython2u;
+select stupidn();
+ stupidn
+---------
+ zarkon
+(1 row)
+
-- test multiple arguments
CREATE FUNCTION argument_test_one(u users, a1 text, a2 text) RETURNS text
AS
diff --git a/src/pl/plpython/sql/plpython_test.sql b/src/pl/plpython/sql/plpython_test.sql
index 161399f..7cae124 100644
--- a/src/pl/plpython/sql/plpython_test.sql
+++ b/src/pl/plpython/sql/plpython_test.sql
@@ -1,10 +1,15 @@
-- first some tests of basic functionality
+CREATE LANGUAGE plpython2u;
-- really stupid function just to get the module loaded
CREATE FUNCTION stupid() RETURNS text AS 'return "zarkon"' LANGUAGE plpythonu;
select stupid();
+-- check 2/3 versioning
+CREATE FUNCTION stupidn() RETURNS text AS 'return "zarkon"' LANGUAGE plpython2u;
+
+select stupidn();
-- test multiple arguments
CREATE FUNCTION argument_test_one(u users, a1 text, a2 text) RETURNS text
On Fri, 2009-12-11 at 01:19 +0200, Peter Eisentraut wrote:
On tor, 2009-11-12 at 16:06 -0500, Tom Lane wrote:
There was considerable debate earlier about whether we wanted to treat
Python 3 as a separate PL so it could be available in parallel with
plpython 2, because of the user-level coding incompatibilities. It
looks like this patch simply ignores that problem. What is going to
happen to plpython functions that depend on 2.x behavior?I have a proposal for how to handle this, and a prototype patch
attached. This follows essentially what the CPython distribution itself
does, which will make this tolerably easy to follow for users.We install plpython as plpython2.so or plpython3.so, depending on the
version used to build it. Then, plpython.so is a symlink to
plpython2.so.We then create three language definition templates:
plpythonu -> plpython.so
plpython2u -> plpython2.so
plpython3u -> plpython3.so
Comments?
Well as a Python guy... makes sense to me :)
Joshua D. Drake
--
PostgreSQL.org Major Contributor
Command Prompt, Inc: http://www.commandprompt.com/ - 503.667.4564
Consulting, Training, Support, Custom Development, Engineering
If the world pushes look it in the eye and GRR. Then push back harder. - Salamander
I wrote:
On tor, 2009-11-12 at 16:06 -0500, Tom Lane wrote:
There was considerable debate earlier about whether we wanted to treat
Python 3 as a separate PL so it could be available in parallel with
plpython 2, because of the user-level coding incompatibilities. It
looks like this patch simply ignores that problem. What is going to
happen to plpython functions that depend on 2.x behavior?I have a proposal for how to handle this, and a prototype patch
attached. This follows essentially what the CPython distribution itself
does, which will make this tolerably easy to follow for users.We install plpython as plpython2.so or plpython3.so, depending on the
version used to build it. Then, plpython.so is a symlink to
plpython2.so.
So here is the potentially final patch for this, including the original
port of plpython.c itself, build system adjustments, and documentation.
On Mon, Dec 14, 2009 at 1:42 PM, Peter Eisentraut <peter_e@gmx.net> wrote:
I wrote:
On tor, 2009-11-12 at 16:06 -0500, Tom Lane wrote:
There was considerable debate earlier about whether we wanted to treat
Python 3 as a separate PL so it could be available in parallel with
plpython 2, because of the user-level coding incompatibilities. It
looks like this patch simply ignores that problem. What is going to
happen to plpython functions that depend on 2.x behavior?I have a proposal for how to handle this, and a prototype patch
attached. This follows essentially what the CPython distribution itself
does, which will make this tolerably easy to follow for users.We install plpython as plpython2.so or plpython3.so, depending on the
version used to build it. Then, plpython.so is a symlink to
plpython2.so.So here is the potentially final patch for this, including the original
port of plpython.c itself, build system adjustments, and documentation.
I think you forgot to actually attach it...
...Robert
I wrote:
On tor, 2009-11-12 at 16:06 -0500, Tom Lane wrote:
There was considerable debate earlier about whether we wanted to treat
Python 3 as a separate PL so it could be available in parallel with
plpython 2, because of the user-level coding incompatibilities. It
looks like this patch simply ignores that problem. What is going to
happen to plpython functions that depend on 2.x behavior?I have a proposal for how to handle this, and a prototype patch
attached. This follows essentially what the CPython distribution itself
does, which will make this tolerably easy to follow for users.We install plpython as plpython2.so or plpython3.so, depending on the
version used to build it. Then, plpython.so is a symlink to
plpython2.so.
So here is the potentially final patch for this, including the original
port of plpython.c itself, build system adjustments, and documentation.
Really attached this time.
Attachments:
python3.1.patchtext/x-patch; charset=UTF-8; name=python3.1.patchDownload
diff --git a/config/python.m4 b/config/python.m4
index 9160a2b..32fff43 100644
--- a/config/python.m4
+++ b/config/python.m4
@@ -30,10 +30,12 @@ else
AC_MSG_ERROR([distutils module not found])
fi
AC_MSG_CHECKING([Python configuration directory])
+python_majorversion=`${PYTHON} -c "import sys; print(sys.version[[0]])"`
python_version=`${PYTHON} -c "import sys; print(sys.version[[:3]])"`
python_configdir=`${PYTHON} -c "from distutils.sysconfig import get_python_lib as f; import os; print(os.path.join(f(plat_specific=1,standard_lib=1),'config'))"`
python_includespec=`${PYTHON} -c "import distutils.sysconfig; print('-I'+distutils.sysconfig.get_python_inc())"`
+AC_SUBST(python_majorversion)[]dnl
AC_SUBST(python_version)[]dnl
AC_SUBST(python_configdir)[]dnl
AC_SUBST(python_includespec)[]dnl
diff --git a/configure b/configure
index 009a177..be51281 100755
--- a/configure
+++ b/configure
@@ -677,6 +677,7 @@ python_libdir
python_includespec
python_configdir
python_version
+python_majorversion
PYTHON
perl_embed_ldflags
perl_useshrplib
@@ -6964,6 +6965,7 @@ $as_echo "$as_me: error: distutils module not found" >&2;}
fi
{ $as_echo "$as_me:$LINENO: checking Python configuration directory" >&5
$as_echo_n "checking Python configuration directory... " >&6; }
+python_majorversion=`${PYTHON} -c "import sys; print(sys.version[0])"`
python_version=`${PYTHON} -c "import sys; print(sys.version[:3])"`
python_configdir=`${PYTHON} -c "from distutils.sysconfig import get_python_lib as f; import os; print(os.path.join(f(plat_specific=1,standard_lib=1),'config'))"`
python_includespec=`${PYTHON} -c "import distutils.sysconfig; print('-I'+distutils.sysconfig.get_python_inc())"`
diff --git a/doc/src/sgml/installation.sgml b/doc/src/sgml/installation.sgml
index 4746e68..f9221c9 100644
--- a/doc/src/sgml/installation.sgml
+++ b/doc/src/sgml/installation.sgml
@@ -195,8 +195,12 @@ su - postgres
<para>
To build the <application>PL/Python</> server programming
language, you need a <productname>Python</productname>
- installation with the header files and the <application>distutils</application> module.
- The minimum required version is <productname>Python</productname> 2.2.
+ installation with the header files and
+ the <application>distutils</application> module. The minimum
+ required version is <productname>Python</productname>
+ 2.2. <productname>Python 3</productname> is supported with
+ version 3.1 or later; but see <xref linkend="plpython-python23">
+ when using Python 3.
</para>
<para>
diff --git a/doc/src/sgml/plpython.sgml b/doc/src/sgml/plpython.sgml
index 502a5bc..e6c998a 100644
--- a/doc/src/sgml/plpython.sgml
+++ b/doc/src/sgml/plpython.sgml
@@ -14,7 +14,8 @@
<para>
To install PL/Python in a particular database, use
- <literal>createlang plpythonu <replaceable>dbname</></literal>.
+ <literal>createlang plpythonu <replaceable>dbname</></literal> (but
+ see also <xref linkend="plpython-python23">).
</para>
<tip>
@@ -42,6 +43,112 @@
</para>
</note>
+ <sect1 id="plpython-python23">
+ <title>Python 2 vs. Python 3</title>
+
+ <para>
+ PL/Python supports both the Python 2 and Python 3 language
+ variants. (The PostgreSQL installation instructions might contain
+ more precise information about the exact supported minor versions
+ of Python.) Because the Python 2 and Python 3 language variants
+ are incompatible in some important aspects, the following naming
+ and transitioning scheme is used by PL/Python to avoid mixing them:
+
+ <itemizedlist>
+ <listitem>
+ <para>
+ The PostgreSQL language named <literal>plpython2u</literal>
+ implements PL/Python based on the Python 2 language variant.
+ </para>
+ </listitem>
+
+ <listitem>
+ <para>
+ The PostgreSQL language named <literal>plpython3u</literal>
+ implements PL/Python based on the Python 3 language variant.
+ </para>
+ </listitem>
+
+ <listitem>
+ <para>
+ The language named <literal>plpythonu</literal> implements
+ PL/Python based on the default Python language variant, which is
+ currently Python 2. (This default is independent of what any
+ local Python installations might consider to be
+ their <quote>default</quote>, for example,
+ what <filename>/usr/bin/python</filename> might be.) The
+ default will probably be changed to Python 3 in a distant future
+ release of PostgreSQL, depending on the progress of the
+ migration to Python 3 in the Python community.
+ </para>
+ </listitem>
+ </itemizedlist>
+
+ It depends on the build configuration or the installed packages
+ whether PL/Python for Python 2 or Python 3 or both are available.
+ </para>
+
+ <para>
+ This results in the following usage and migration strategy:
+
+ <itemizedlist>
+ <listitem>
+ <para>
+ Existing users and users who are currently not interested in
+ Python 3 use the language name <literal>plpythonu</literal> and
+ don't have to change anything for the foreseeable future. It is
+ recommended to gradually <quote>future-proof</quote> the code
+ via migration to Python 2.6/2.7 to simplify the eventual
+ migration to Python 3.
+ </para>
+
+ <para>
+ In practice, many PL/Python functions will migrate to Python 3
+ with few or no changes.
+ </para>
+ </listitem>
+
+ <listitem>
+ <para>
+ Users who know that they have heavily Python 2 dependent code
+ and don't plan to ever change it can make use of
+ the <literal>plpython2u</literal> language name. This will
+ continue to work into the very distant future, until Python 2
+ support might be completely dropped by PostgreSQL.
+ </para>
+ </listitem>
+
+ <listitem>
+ <para>
+ Users who want to dive into Python 3 can use
+ the <literal>plpython3u</literal> language name, which will keep
+ working forever by today's standards. In the distant future,
+ when Python 3 might become the default, they might like to
+ remove the <quote>3</quote> for aesthetic reasons.
+ </para>
+ </listitem>
+
+ <listitem>
+ <para>
+ Daredevils, who want to build a Python-3-only operating system
+ environment, can change the build scripts to
+ make <literal>plpythonu</literal> be equivalent
+ to <literal>plpython3u</literal>, keeping in mind that this
+ would make their installation incompatible with most of the rest
+ of the world.
+ </para>
+ </listitem>
+ </itemizedlist>
+ </para>
+
+ <para>
+ See also the
+ document <ulink url="http://docs.python.org/dev/3.0/whatsnew/3.0.html">What's
+ New In Python 3.0</ulink> for more information about porting to
+ Python 3.
+ </para>
+ </sect1>
+
<sect1 id="plpython-funcs">
<title>PL/Python Functions</title>
diff --git a/src/Makefile.global.in b/src/Makefile.global.in
index ca7f499..7a6e3a9 100644
--- a/src/Makefile.global.in
+++ b/src/Makefile.global.in
@@ -171,6 +171,7 @@ python_libdir = @python_libdir@
python_libspec = @python_libspec@
python_additional_libs = @python_additional_libs@
python_configdir = @python_configdir@
+python_majorversion = @python_majorversion@
python_version = @python_version@
krb_srvtab = @krb_srvtab@
diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h
index a2d1859..e0bd98b 100644
--- a/src/include/catalog/catversion.h
+++ b/src/include/catalog/catversion.h
@@ -53,6 +53,6 @@
*/
/* yyyymmddN */
-#define CATALOG_VERSION_NO 200912111
+#define CATALOG_VERSION_NO 200912131
#endif
diff --git a/src/include/catalog/pg_pltemplate.h b/src/include/catalog/pg_pltemplate.h
index 8cdedb4..cbb0a33 100644
--- a/src/include/catalog/pg_pltemplate.h
+++ b/src/include/catalog/pg_pltemplate.h
@@ -73,5 +73,7 @@ DATA(insert ( "pltclu" f f "pltclu_call_handler" _null_ _null_ "$libdir/pltcl"
DATA(insert ( "plperl" t t "plperl_call_handler" "plperl_inline_handler" "plperl_validator" "$libdir/plperl" _null_ ));
DATA(insert ( "plperlu" f f "plperl_call_handler" "plperl_inline_handler" "plperl_validator" "$libdir/plperl" _null_ ));
DATA(insert ( "plpythonu" f f "plpython_call_handler" _null_ _null_ "$libdir/plpython" _null_ ));
+DATA(insert ( "plpython2u" f f "plpython_call_handler" _null_ _null_ "$libdir/plpython2" _null_ ));
+DATA(insert ( "plpython3u" f f "plpython_call_handler" _null_ _null_ "$libdir/plpython3" _null_ ));
#endif /* PG_PLTEMPLATE_H */
diff --git a/src/pl/plpython/Makefile b/src/pl/plpython/Makefile
index 373bc79..50b8ea1 100644
--- a/src/pl/plpython/Makefile
+++ b/src/pl/plpython/Makefile
@@ -36,7 +36,7 @@ override CPPFLAGS := -I$(srcdir) $(python_includespec) $(CPPFLAGS)
rpathdir = $(python_libdir)
-NAME = plpython
+NAME = plpython$(python_majorversion)
OBJS = plpython.o
@@ -56,7 +56,12 @@ endif
SHLIB_LINK = $(python_libspec) $(python_additional_libs) $(filter -lintl,$(LIBS))
-REGRESS_OPTS = --dbname=$(PL_TESTDB) --load-language=plpythonu
+REGRESS_OPTS = --dbname=$(PL_TESTDB)
+# Only load plpythonu with Python 2. The test files themselves load
+# the versioned language plpython(2|3)u.
+ifeq ($(python_majorversion),2)
+REGRESS_OPTS += --load-language=plpythonu
+endif
REGRESS = \
plpython_schema \
plpython_populate \
@@ -83,13 +88,45 @@ include $(top_srcdir)/src/Makefile.shlib
all: all-lib
install: all installdirs install-lib
+ifeq ($(python_majorversion),2)
+ cd '$(DESTDIR)$(pkglibdir)' && rm -f plpython$(DLSUFFIX) && $(LN_S) $(shlib) plpython$(DLSUFFIX)
+endif
installdirs: installdirs-lib
uninstall: uninstall-lib
+ifeq ($(python_majorversion),2)
+ rm -f '$(DESTDIR)$(pkglibdir)/plpython$(DLSUFFIX)'
+endif
+ifeq ($(python_majorversion),3)
+# Adjust regression tests for Python 3 compatibility
+prep3:
+ $(MKDIR_P) python3 python3/sql python3/expected
+ for file in $(srcdir)/sql/* $(srcdir)/expected/*; do \
+ sed -e 's/except \([[:alpha:]][[:alpha:].]*\), *\([[:alpha:]][[:alpha:]]*\):/except \1 as \2:/g' \
+ -e "s/<type 'exceptions\.\([[:alpha:]]*\)'>/<class '\1'>/g" \
+ -e "s/<type 'long'>/<class 'int'>/g" \
+ -e "s/\([0-9][0-9]*\)L/\1/g" \
+ -e 's/\([ [{]\)u"/\1"/g' \
+ -e "s/\([ [{]\)u'/\1'/g" \
+ -e "s/def next/def __next__/g" \
+ -e "s/LANGUAGE plpythonu/LANGUAGE plpython3u/g" \
+ -e "s/LANGUAGE plpython2u/LANGUAGE plpython3u/g" \
+ $$file >`echo $$file | sed 's,$(srcdir),python3,'`; \
+ done
+
+clean3:
+ rm -rf python3/
+
+installcheck: submake prep3
+ $(top_builddir)/src/test/regress/pg_regress --inputdir=./python3 --outputdir=./python3 --psqldir=$(PSQLDIR) $(REGRESS_OPTS) $(REGRESS)
+
+clean: clean3
+else
installcheck: submake
$(top_builddir)/src/test/regress/pg_regress --inputdir=$(srcdir) --psqldir=$(PSQLDIR) $(REGRESS_OPTS) $(REGRESS)
+endif
.PHONY: submake
submake:
diff --git a/src/pl/plpython/expected/README b/src/pl/plpython/expected/README
index 47f31e8..a187937 100644
--- a/src/pl/plpython/expected/README
+++ b/src/pl/plpython/expected/README
@@ -8,3 +8,5 @@ plpython_unicode_0.out any version, when server encoding != SQL_ASCII and clien
plpython_unicode_2.out Python 2.2
plpython_unicode_3.out Python 2.3, 2.4
plpython_unicode_5.out Python 2.5, 2.6
+
+plpython_types_3.out Python 3.1
diff --git a/src/pl/plpython/expected/plpython_test.out b/src/pl/plpython/expected/plpython_test.out
index c5cfe5a..a229b18 100644
--- a/src/pl/plpython/expected/plpython_test.out
+++ b/src/pl/plpython/expected/plpython_test.out
@@ -1,4 +1,5 @@
-- first some tests of basic functionality
+CREATE LANGUAGE plpython2u;
-- really stupid function just to get the module loaded
CREATE FUNCTION stupid() RETURNS text AS 'return "zarkon"' LANGUAGE plpythonu;
select stupid();
@@ -7,6 +8,14 @@ select stupid();
zarkon
(1 row)
+-- check 2/3 versioning
+CREATE FUNCTION stupidn() RETURNS text AS 'return "zarkon"' LANGUAGE plpython2u;
+select stupidn();
+ stupidn
+---------
+ zarkon
+(1 row)
+
-- test multiple arguments
CREATE FUNCTION argument_test_one(u users, a1 text, a2 text) RETURNS text
AS
diff --git a/src/pl/plpython/expected/plpython_trigger.out b/src/pl/plpython/expected/plpython_trigger.out
index cf5c758..3192ff1 100644
--- a/src/pl/plpython/expected/plpython_trigger.out
+++ b/src/pl/plpython/expected/plpython_trigger.out
@@ -67,7 +67,7 @@ SELECT * FROM users;
-- dump trigger data
CREATE TABLE trigger_test
(i int, v text );
-CREATE FUNCTION trigger_data() returns trigger language plpythonu as $$
+CREATE FUNCTION trigger_data() RETURNS trigger LANGUAGE plpythonu AS $$
if 'relid' in TD:
TD['relid'] = "bogus:12345"
diff --git a/src/pl/plpython/expected/plpython_types_3.out b/src/pl/plpython/expected/plpython_types_3.out
new file mode 100644
index 0000000..3fcb0f4
--- /dev/null
+++ b/src/pl/plpython/expected/plpython_types_3.out
@@ -0,0 +1,479 @@
+--
+-- Test data type behavior
+--
+--
+-- Base/common types
+--
+CREATE FUNCTION test_type_conversion_bool(x bool) RETURNS bool AS $$
+plpy.info(x, type(x))
+return x
+$$ LANGUAGE plpythonu;
+SELECT * FROM test_type_conversion_bool(true);
+INFO: (True, <class 'bool'>)
+CONTEXT: PL/Python function "test_type_conversion_bool"
+ test_type_conversion_bool
+---------------------------
+ t
+(1 row)
+
+SELECT * FROM test_type_conversion_bool(false);
+INFO: (False, <class 'bool'>)
+CONTEXT: PL/Python function "test_type_conversion_bool"
+ test_type_conversion_bool
+---------------------------
+ f
+(1 row)
+
+SELECT * FROM test_type_conversion_bool(null);
+INFO: (None, <class 'NoneType'>)
+CONTEXT: PL/Python function "test_type_conversion_bool"
+ test_type_conversion_bool
+---------------------------
+
+(1 row)
+
+-- test various other ways to express Booleans in Python
+CREATE FUNCTION test_type_conversion_bool_other(n int) RETURNS bool AS $$
+# numbers
+if n == 0:
+ ret = 0
+elif n == 1:
+ ret = 5
+# strings
+elif n == 2:
+ ret = ''
+elif n == 3:
+ ret = 'fa' # true in Python, false in PostgreSQL
+# containers
+elif n == 4:
+ ret = []
+elif n == 5:
+ ret = [0]
+plpy.info(ret, not not ret)
+return ret
+$$ LANGUAGE plpythonu;
+SELECT * FROM test_type_conversion_bool_other(0);
+INFO: (0, False)
+CONTEXT: PL/Python function "test_type_conversion_bool_other"
+ test_type_conversion_bool_other
+---------------------------------
+ f
+(1 row)
+
+SELECT * FROM test_type_conversion_bool_other(1);
+INFO: (5, True)
+CONTEXT: PL/Python function "test_type_conversion_bool_other"
+ test_type_conversion_bool_other
+---------------------------------
+ t
+(1 row)
+
+SELECT * FROM test_type_conversion_bool_other(2);
+INFO: ('', False)
+CONTEXT: PL/Python function "test_type_conversion_bool_other"
+ test_type_conversion_bool_other
+---------------------------------
+ f
+(1 row)
+
+SELECT * FROM test_type_conversion_bool_other(3);
+INFO: ('fa', True)
+CONTEXT: PL/Python function "test_type_conversion_bool_other"
+ test_type_conversion_bool_other
+---------------------------------
+ t
+(1 row)
+
+SELECT * FROM test_type_conversion_bool_other(4);
+INFO: ([], False)
+CONTEXT: PL/Python function "test_type_conversion_bool_other"
+ test_type_conversion_bool_other
+---------------------------------
+ f
+(1 row)
+
+SELECT * FROM test_type_conversion_bool_other(5);
+INFO: ([0], True)
+CONTEXT: PL/Python function "test_type_conversion_bool_other"
+ test_type_conversion_bool_other
+---------------------------------
+ t
+(1 row)
+
+CREATE FUNCTION test_type_conversion_char(x char) RETURNS char AS $$
+plpy.info(x, type(x))
+return x
+$$ LANGUAGE plpythonu;
+SELECT * FROM test_type_conversion_char('a');
+INFO: ('a', <class 'str'>)
+CONTEXT: PL/Python function "test_type_conversion_char"
+ test_type_conversion_char
+---------------------------
+ a
+(1 row)
+
+SELECT * FROM test_type_conversion_char(null);
+INFO: (None, <class 'NoneType'>)
+CONTEXT: PL/Python function "test_type_conversion_char"
+ test_type_conversion_char
+---------------------------
+
+(1 row)
+
+CREATE FUNCTION test_type_conversion_int2(x int2) RETURNS int2 AS $$
+plpy.info(x, type(x))
+return x
+$$ LANGUAGE plpythonu;
+SELECT * FROM test_type_conversion_int2(100::int2);
+INFO: (100, <class 'int'>)
+CONTEXT: PL/Python function "test_type_conversion_int2"
+ test_type_conversion_int2
+---------------------------
+ 100
+(1 row)
+
+SELECT * FROM test_type_conversion_int2(-100::int2);
+INFO: (-100, <class 'int'>)
+CONTEXT: PL/Python function "test_type_conversion_int2"
+ test_type_conversion_int2
+---------------------------
+ -100
+(1 row)
+
+SELECT * FROM test_type_conversion_int2(null);
+INFO: (None, <class 'NoneType'>)
+CONTEXT: PL/Python function "test_type_conversion_int2"
+ test_type_conversion_int2
+---------------------------
+
+(1 row)
+
+CREATE FUNCTION test_type_conversion_int4(x int4) RETURNS int4 AS $$
+plpy.info(x, type(x))
+return x
+$$ LANGUAGE plpythonu;
+SELECT * FROM test_type_conversion_int4(100);
+INFO: (100, <class 'int'>)
+CONTEXT: PL/Python function "test_type_conversion_int4"
+ test_type_conversion_int4
+---------------------------
+ 100
+(1 row)
+
+SELECT * FROM test_type_conversion_int4(-100);
+INFO: (-100, <class 'int'>)
+CONTEXT: PL/Python function "test_type_conversion_int4"
+ test_type_conversion_int4
+---------------------------
+ -100
+(1 row)
+
+SELECT * FROM test_type_conversion_int4(null);
+INFO: (None, <class 'NoneType'>)
+CONTEXT: PL/Python function "test_type_conversion_int4"
+ test_type_conversion_int4
+---------------------------
+
+(1 row)
+
+CREATE FUNCTION test_type_conversion_int8(x int8) RETURNS int8 AS $$
+plpy.info(x, type(x))
+return x
+$$ LANGUAGE plpythonu;
+SELECT * FROM test_type_conversion_int8(100);
+INFO: (100, <class 'int'>)
+CONTEXT: PL/Python function "test_type_conversion_int8"
+ test_type_conversion_int8
+---------------------------
+ 100
+(1 row)
+
+SELECT * FROM test_type_conversion_int8(-100);
+INFO: (-100, <class 'int'>)
+CONTEXT: PL/Python function "test_type_conversion_int8"
+ test_type_conversion_int8
+---------------------------
+ -100
+(1 row)
+
+SELECT * FROM test_type_conversion_int8(5000000000);
+INFO: (5000000000, <class 'int'>)
+CONTEXT: PL/Python function "test_type_conversion_int8"
+ test_type_conversion_int8
+---------------------------
+ 5000000000
+(1 row)
+
+SELECT * FROM test_type_conversion_int8(null);
+INFO: (None, <class 'NoneType'>)
+CONTEXT: PL/Python function "test_type_conversion_int8"
+ test_type_conversion_int8
+---------------------------
+
+(1 row)
+
+CREATE FUNCTION test_type_conversion_numeric(x numeric) RETURNS numeric AS $$
+plpy.info(x, type(x))
+return x
+$$ LANGUAGE plpythonu;
+/* The current implementation converts numeric to float. */
+SELECT * FROM test_type_conversion_numeric(100);
+INFO: (100.0, <class 'float'>)
+CONTEXT: PL/Python function "test_type_conversion_numeric"
+ test_type_conversion_numeric
+------------------------------
+ 100.0
+(1 row)
+
+SELECT * FROM test_type_conversion_numeric(-100);
+INFO: (-100.0, <class 'float'>)
+CONTEXT: PL/Python function "test_type_conversion_numeric"
+ test_type_conversion_numeric
+------------------------------
+ -100.0
+(1 row)
+
+SELECT * FROM test_type_conversion_numeric(5000000000.5);
+INFO: (5000000000.5, <class 'float'>)
+CONTEXT: PL/Python function "test_type_conversion_numeric"
+ test_type_conversion_numeric
+------------------------------
+ 5000000000.5
+(1 row)
+
+SELECT * FROM test_type_conversion_numeric(null);
+INFO: (None, <class 'NoneType'>)
+CONTEXT: PL/Python function "test_type_conversion_numeric"
+ test_type_conversion_numeric
+------------------------------
+
+(1 row)
+
+CREATE FUNCTION test_type_conversion_float4(x float4) RETURNS float4 AS $$
+plpy.info(x, type(x))
+return x
+$$ LANGUAGE plpythonu;
+SELECT * FROM test_type_conversion_float4(100);
+INFO: (100.0, <class 'float'>)
+CONTEXT: PL/Python function "test_type_conversion_float4"
+ test_type_conversion_float4
+-----------------------------
+ 100
+(1 row)
+
+SELECT * FROM test_type_conversion_float4(-100);
+INFO: (-100.0, <class 'float'>)
+CONTEXT: PL/Python function "test_type_conversion_float4"
+ test_type_conversion_float4
+-----------------------------
+ -100
+(1 row)
+
+SELECT * FROM test_type_conversion_float4(5000.5);
+INFO: (5000.5, <class 'float'>)
+CONTEXT: PL/Python function "test_type_conversion_float4"
+ test_type_conversion_float4
+-----------------------------
+ 5000.5
+(1 row)
+
+SELECT * FROM test_type_conversion_float4(null);
+INFO: (None, <class 'NoneType'>)
+CONTEXT: PL/Python function "test_type_conversion_float4"
+ test_type_conversion_float4
+-----------------------------
+
+(1 row)
+
+CREATE FUNCTION test_type_conversion_float8(x float8) RETURNS float8 AS $$
+plpy.info(x, type(x))
+return x
+$$ LANGUAGE plpythonu;
+SELECT * FROM test_type_conversion_float8(100);
+INFO: (100.0, <class 'float'>)
+CONTEXT: PL/Python function "test_type_conversion_float8"
+ test_type_conversion_float8
+-----------------------------
+ 100
+(1 row)
+
+SELECT * FROM test_type_conversion_float8(-100);
+INFO: (-100.0, <class 'float'>)
+CONTEXT: PL/Python function "test_type_conversion_float8"
+ test_type_conversion_float8
+-----------------------------
+ -100
+(1 row)
+
+SELECT * FROM test_type_conversion_float8(5000000000.5);
+INFO: (5000000000.5, <class 'float'>)
+CONTEXT: PL/Python function "test_type_conversion_float8"
+ test_type_conversion_float8
+-----------------------------
+ 5000000000.5
+(1 row)
+
+SELECT * FROM test_type_conversion_float8(null);
+INFO: (None, <class 'NoneType'>)
+CONTEXT: PL/Python function "test_type_conversion_float8"
+ test_type_conversion_float8
+-----------------------------
+
+(1 row)
+
+CREATE FUNCTION test_type_conversion_text(x text) RETURNS text AS $$
+plpy.info(x, type(x))
+return x
+$$ LANGUAGE plpythonu;
+SELECT * FROM test_type_conversion_text('hello world');
+INFO: ('hello world', <class 'str'>)
+CONTEXT: PL/Python function "test_type_conversion_text"
+ test_type_conversion_text
+---------------------------
+ hello world
+(1 row)
+
+SELECT * FROM test_type_conversion_text(null);
+INFO: (None, <class 'NoneType'>)
+CONTEXT: PL/Python function "test_type_conversion_text"
+ test_type_conversion_text
+---------------------------
+
+(1 row)
+
+CREATE FUNCTION test_type_conversion_bytea(x bytea) RETURNS bytea AS $$
+plpy.info(x, type(x))
+return x
+$$ LANGUAGE plpythonu;
+SELECT * FROM test_type_conversion_bytea('hello world');
+INFO: (b'hello world', <class 'bytes'>)
+CONTEXT: PL/Python function "test_type_conversion_bytea"
+ test_type_conversion_bytea
+----------------------------
+ \x68656c6c6f20776f726c64
+(1 row)
+
+SELECT * FROM test_type_conversion_bytea(E'null\\000byte');
+INFO: (b'null\x00byte', <class 'bytes'>)
+CONTEXT: PL/Python function "test_type_conversion_bytea"
+ test_type_conversion_bytea
+----------------------------
+ \x6e756c6c0062797465
+(1 row)
+
+SELECT * FROM test_type_conversion_bytea(null);
+INFO: (None, <class 'NoneType'>)
+CONTEXT: PL/Python function "test_type_conversion_bytea"
+ test_type_conversion_bytea
+----------------------------
+
+(1 row)
+
+CREATE FUNCTION test_type_marshal() RETURNS bytea AS $$
+import marshal
+return marshal.dumps('hello world')
+$$ LANGUAGE plpythonu;
+CREATE FUNCTION test_type_unmarshal(x bytea) RETURNS text AS $$
+import marshal
+try:
+ return marshal.loads(x)
+except ValueError as e:
+ return 'FAILED: ' + str(e)
+$$ LANGUAGE plpythonu;
+SELECT test_type_unmarshal(x) FROM test_type_marshal() x;
+ test_type_unmarshal
+---------------------
+ hello world
+(1 row)
+
+--
+-- Domains
+--
+CREATE DOMAIN booltrue AS bool CHECK (VALUE IS TRUE OR VALUE IS NULL);
+CREATE FUNCTION test_type_conversion_booltrue(x booltrue, y bool) RETURNS booltrue AS $$
+return y
+$$ LANGUAGE plpythonu;
+SELECT * FROM test_type_conversion_booltrue(true, true);
+ test_type_conversion_booltrue
+-------------------------------
+ t
+(1 row)
+
+SELECT * FROM test_type_conversion_booltrue(false, true);
+ERROR: value for domain booltrue violates check constraint "booltrue_check"
+SELECT * FROM test_type_conversion_booltrue(true, false);
+ERROR: value for domain booltrue violates check constraint "booltrue_check"
+CONTEXT: while creating return value
+PL/Python function "test_type_conversion_booltrue"
+CREATE DOMAIN uint2 AS int2 CHECK (VALUE >= 0);
+CREATE FUNCTION test_type_conversion_uint2(x uint2, y int) RETURNS uint2 AS $$
+plpy.info(x, type(x))
+return y
+$$ LANGUAGE plpythonu;
+SELECT * FROM test_type_conversion_uint2(100::uint2, 50);
+INFO: (100, <class 'int'>)
+CONTEXT: PL/Python function "test_type_conversion_uint2"
+ test_type_conversion_uint2
+----------------------------
+ 50
+(1 row)
+
+SELECT * FROM test_type_conversion_uint2(100::uint2, -50);
+INFO: (100, <class 'int'>)
+CONTEXT: PL/Python function "test_type_conversion_uint2"
+ERROR: value for domain uint2 violates check constraint "uint2_check"
+CONTEXT: while creating return value
+PL/Python function "test_type_conversion_uint2"
+SELECT * FROM test_type_conversion_uint2(null, 1);
+INFO: (None, <class 'NoneType'>)
+CONTEXT: PL/Python function "test_type_conversion_uint2"
+ test_type_conversion_uint2
+----------------------------
+ 1
+(1 row)
+
+CREATE DOMAIN nnint AS int CHECK (VALUE IS NOT NULL);
+CREATE FUNCTION test_type_conversion_nnint(x nnint, y int) RETURNS nnint AS $$
+return y
+$$ LANGUAGE plpythonu;
+SELECT * FROM test_type_conversion_nnint(10, 20);
+ test_type_conversion_nnint
+----------------------------
+ 20
+(1 row)
+
+SELECT * FROM test_type_conversion_nnint(null, 20);
+ERROR: value for domain nnint violates check constraint "nnint_check"
+SELECT * FROM test_type_conversion_nnint(10, null);
+ERROR: value for domain nnint violates check constraint "nnint_check"
+CONTEXT: while creating return value
+PL/Python function "test_type_conversion_nnint"
+CREATE DOMAIN bytea10 AS bytea CHECK (octet_length(VALUE) = 10 AND VALUE IS NOT NULL);
+CREATE FUNCTION test_type_conversion_bytea10(x bytea10, y bytea) RETURNS bytea10 AS $$
+plpy.info(x, type(x))
+return y
+$$ LANGUAGE plpythonu;
+SELECT * FROM test_type_conversion_bytea10('hello wold', 'hello wold');
+INFO: (b'hello wold', <class 'bytes'>)
+CONTEXT: PL/Python function "test_type_conversion_bytea10"
+ test_type_conversion_bytea10
+------------------------------
+ \x68656c6c6f20776f6c64
+(1 row)
+
+SELECT * FROM test_type_conversion_bytea10('hello world', 'hello wold');
+ERROR: value for domain bytea10 violates check constraint "bytea10_check"
+SELECT * FROM test_type_conversion_bytea10('hello word', 'hello world');
+INFO: (b'hello word', <class 'bytes'>)
+CONTEXT: PL/Python function "test_type_conversion_bytea10"
+ERROR: value for domain bytea10 violates check constraint "bytea10_check"
+CONTEXT: while creating return value
+PL/Python function "test_type_conversion_bytea10"
+SELECT * FROM test_type_conversion_bytea10(null, 'hello word');
+ERROR: value for domain bytea10 violates check constraint "bytea10_check"
+SELECT * FROM test_type_conversion_bytea10('hello word', null);
+INFO: (b'hello word', <class 'bytes'>)
+CONTEXT: PL/Python function "test_type_conversion_bytea10"
+ERROR: value for domain bytea10 violates check constraint "bytea10_check"
+CONTEXT: while creating return value
+PL/Python function "test_type_conversion_bytea10"
diff --git a/src/pl/plpython/plpython.c b/src/pl/plpython/plpython.c
index 6a2a12f..fd69953 100644
--- a/src/pl/plpython/plpython.c
+++ b/src/pl/plpython/plpython.c
@@ -40,6 +40,48 @@ typedef int Py_ssize_t;
#define PyBool_FromLong(x) PyInt_FromLong(x)
#endif
+/*
+ * Python 2/3 strings/unicode/bytes handling. Python 2 has strings
+ * and unicode, Python 3 has strings, which are unicode on the C
+ * level, and bytes. The porting convention, which is similarly used
+ * in Python 2.6, is that "Unicode" is always unicode, and "Bytes" are
+ * bytes in Python 3 and strings in Python 2. Since we keep
+ * supporting Python 2 and its usual strings, we provide a
+ * compatibility layer for Python 3 that when asked to convert a C
+ * string to a Python string it converts the C string from the
+ * PostgreSQL server encoding to a Python Unicode object.
+ */
+
+#if PY_VERSION_HEX < 0x02060000
+/* This is exactly the compatibility layer that Python 2.6 uses. */
+#define PyBytes_AsString PyString_AsString
+#define PyBytes_FromStringAndSize PyString_FromStringAndSize
+#define PyBytes_Size PyString_Size
+#define PyObject_Bytes PyObject_Str
+#endif
+
+#if PY_MAJOR_VERSION >= 3
+#define PyString_Check(x) 0
+#define PyString_AsString(x) PLyUnicode_AsString(x)
+#define PyString_FromString(x) PLyUnicode_FromString(x)
+#endif
+
+/*
+ * Python 3 only has long.
+ */
+#if PY_MAJOR_VERSION >= 3
+#define PyInt_FromLong(x) PyLong_FromLong(x)
+#endif
+
+/*
+ * PyVarObject_HEAD_INIT was added in Python 2.6. Its use is
+ * necessary to handle both Python 2 and 3. This replacement
+ * definition is for Python <=2.5
+ */
+#ifndef PyVarObject_HEAD_INIT
+#define PyVarObject_HEAD_INIT(type, size) \
+ PyObject_HEAD_INIT(type) size,
+#endif
#include "postgres.h"
@@ -246,7 +288,11 @@ static char *PLy_strdup(const char *);
static void PLy_free(void *);
static PyObject*PLyUnicode_Str(PyObject *unicode);
+static PyObject*PLyUnicode_Bytes(PyObject *unicode);
static char *PLyUnicode_AsString(PyObject *unicode);
+#if PY_MAJOR_VERSION >= 3
+static PyObject *PLyUnicode_FromString(const char *s);
+#endif
/* sub handlers for functions and triggers */
static Datum PLy_function_handler(FunctionCallInfo fcinfo, PLyProcedure *);
@@ -288,7 +334,7 @@ static PyObject *PLyFloat_FromNumeric(PLyDatumToOb *arg, Datum d);
static PyObject *PLyInt_FromInt16(PLyDatumToOb *arg, Datum d);
static PyObject *PLyInt_FromInt32(PLyDatumToOb *arg, Datum d);
static PyObject *PLyLong_FromInt64(PLyDatumToOb *arg, Datum d);
-static PyObject *PLyString_FromBytea(PLyDatumToOb *arg, Datum d);
+static PyObject *PLyBytes_FromBytea(PLyDatumToOb *arg, Datum d);
static PyObject *PLyString_FromDatum(PLyDatumToOb *arg, Datum d);
static PyObject *PLyList_FromArray(PLyDatumToOb *arg, Datum d);
@@ -1760,7 +1806,7 @@ PLy_input_datum_func2(PLyDatumToOb *arg, Oid typeOid, HeapTuple typeTup)
arg->func = PLyLong_FromInt64;
break;
case BYTEAOID:
- arg->func = PLyString_FromBytea;
+ arg->func = PLyBytes_FromBytea;
break;
default:
arg->func = PLyString_FromDatum;
@@ -1859,13 +1905,13 @@ PLyLong_FromInt64(PLyDatumToOb *arg, Datum d)
}
static PyObject *
-PLyString_FromBytea(PLyDatumToOb *arg, Datum d)
+PLyBytes_FromBytea(PLyDatumToOb *arg, Datum d)
{
text *txt = DatumGetByteaP(d);
char *str = VARDATA(txt);
size_t size = VARSIZE(txt) - VARHDRSZ;
- return PyString_FromStringAndSize(str, size);
+ return PyBytes_FromStringAndSize(str, size);
}
static PyObject *
@@ -2001,14 +2047,14 @@ PLyObject_ToBytea(PLyTypeInfo *info,
Assert(plrv != Py_None);
- plrv_so = PyObject_Str(plrv);
+ plrv_so = PyObject_Bytes(plrv);
if (!plrv_so)
- PLy_elog(ERROR, "could not create string representation of Python object");
+ PLy_elog(ERROR, "could not create bytes representation of Python object");
PG_TRY();
{
- char *plrv_sc = PyString_AsString(plrv_so);
- size_t len = PyString_Size(plrv_so);
+ char *plrv_sc = PyBytes_AsString(plrv_so);
+ size_t len = PyBytes_Size(plrv_so);
size_t size = len + VARHDRSZ;
bytea *result = palloc(size);
@@ -2040,22 +2086,30 @@ PLyObject_ToDatum(PLyTypeInfo *info,
PLyObToDatum *arg,
PyObject *plrv)
{
- PyObject *volatile plrv_so = NULL;
+ PyObject *volatile plrv_bo = NULL;
Datum rv;
Assert(plrv != Py_None);
if (PyUnicode_Check(plrv))
- plrv_so = PLyUnicode_Str(plrv);
+ plrv_bo = PLyUnicode_Bytes(plrv);
else
- plrv_so = PyObject_Str(plrv);
- if (!plrv_so)
+ {
+#if PY_MAJOR_VERSION >= 3
+ PyObject *s = PyObject_Str(plrv);
+ plrv_bo = PLyUnicode_Bytes(s);
+ Py_XDECREF(s);
+#else
+ plrv_bo = PyObject_Str(plrv);
+#endif
+ }
+ if (!plrv_bo)
PLy_elog(ERROR, "could not create string representation of Python object");
PG_TRY();
{
- char *plrv_sc = PyString_AsString(plrv_so);
- size_t plen = PyString_Size(plrv_so);
+ char *plrv_sc = PyBytes_AsString(plrv_bo);
+ size_t plen = PyBytes_Size(plrv_bo);
size_t slen = strlen(plrv_sc);
if (slen < plen)
@@ -2068,12 +2122,12 @@ PLyObject_ToDatum(PLyTypeInfo *info,
}
PG_CATCH();
{
- Py_XDECREF(plrv_so);
+ Py_XDECREF(plrv_bo);
PG_RE_THROW();
}
PG_END_TRY();
- Py_XDECREF(plrv_so);
+ Py_XDECREF(plrv_bo);
return rv;
}
@@ -2368,8 +2422,7 @@ static PyMethodDef PLy_plan_methods[] = {
};
static PyTypeObject PLy_PlanType = {
- PyObject_HEAD_INIT(NULL)
- 0, /* ob_size */
+ PyVarObject_HEAD_INIT(NULL, 0)
"PLyPlan", /* tp_name */
sizeof(PLyPlanObject), /* tp_size */
0, /* tp_itemsize */
@@ -2420,8 +2473,7 @@ static PyMethodDef PLy_result_methods[] = {
};
static PyTypeObject PLy_ResultType = {
- PyObject_HEAD_INIT(NULL)
- 0, /* ob_size */
+ PyVarObject_HEAD_INIT(NULL, 0)
"PLyResult", /* tp_name */
sizeof(PLyResultObject), /* tp_size */
0, /* tp_itemsize */
@@ -2480,6 +2532,15 @@ static PyMethodDef PLy_methods[] = {
{NULL, NULL, 0, NULL}
};
+#if PY_MAJOR_VERSION >= 3
+static PyModuleDef PLy_module = {
+ PyModuleDef_HEAD_INIT, /* m_base */
+ "plpy", /* m_name */
+ NULL, /* m_doc */
+ -1, /* m_size */
+ PLy_methods, /* m_methods */
+};
+#endif
/* plan object methods */
static PyObject *
@@ -3067,6 +3128,15 @@ PLy_spi_execute_fetch_result(SPITupleTable *tuptable, int rows, int status)
* language handler and interpreter initialization
*/
+#if PY_MAJOR_VERSION >= 3
+static PyMODINIT_FUNC
+PyInit_plpy(void)
+{
+ return PyModule_Create(&PLy_module);
+}
+#endif
+
+
/*
* _PG_init() - library load-time initialization
*
@@ -3083,7 +3153,13 @@ _PG_init(void)
pg_bindtextdomain(TEXTDOMAIN);
+#if PY_MAJOR_VERSION >= 3
+ PyImport_AppendInittab("plpy", PyInit_plpy);
+#endif
Py_Initialize();
+#if PY_MAJOR_VERSION >= 3
+ PyImport_ImportModule("plpy");
+#endif
PLy_init_interp();
PLy_init_plpy();
if (PyErr_Occurred())
@@ -3129,7 +3205,11 @@ PLy_init_plpy(void)
if (PyType_Ready(&PLy_ResultType) < 0)
elog(ERROR, "could not initialize PLy_ResultType");
+#if PY_MAJOR_VERSION >= 3
+ plpy = PyModule_Create(&PLy_module);
+#else
plpy = Py_InitModule("plpy", PLy_methods);
+#endif
plpy_dict = PyModule_GetDict(plpy);
/* PyDict_SetItemString(plpy, "PlanType", (PyObject *) &PLy_PlanType); */
@@ -3475,12 +3555,29 @@ PLy_free(void *ptr)
}
/*
- * Convert a Python unicode object to a Python string object in
+ * Convert a Unicode object to a Python string.
+ */
+static PyObject*
+PLyUnicode_Str(PyObject *unicode)
+{
+#if PY_MAJOR_VERSION >= 3
+ /* In Python 3, this is a noop. */
+ Py_INCREF(unicode);
+ return unicode;
+#else
+ /* In Python 2, this means converting the Unicode to bytes in the
+ * server encoding. */
+ return PLyUnicode_Bytes(unicode);
+#endif
+}
+
+/*
+ * Convert a Python unicode object to a Python string/bytes object in
* PostgreSQL server encoding. Reference ownership is passed to the
* caller.
*/
static PyObject*
-PLyUnicode_Str(PyObject *unicode)
+PLyUnicode_Bytes(PyObject *unicode)
{
PyObject *rv;
const char *serverenc;
@@ -3502,13 +3599,44 @@ PLyUnicode_Str(PyObject *unicode)
/*
* Convert a Python unicode object to a C string in PostgreSQL server
* encoding. No Python object reference is passed out of this
- * function.
+ * function. The result is palloc'ed.
+ *
+ * Note that this function is disguised as PyString_AsString() when
+ * using Python 3. That function retuns a pointer into the internal
+ * memory of the argument, which isn't exactly the interface of this
+ * function. But in either case you get a rather short-lived
+ * reference that you ought to better leave alone.
*/
static char *
PLyUnicode_AsString(PyObject *unicode)
{
- PyObject *o = PLyUnicode_Str(unicode);
- char *rv = PyString_AsString(o);
+ PyObject *o = PLyUnicode_Bytes(unicode);
+ char *rv = pstrdup(PyBytes_AsString(o));
Py_XDECREF(o);
return rv;
}
+
+#if PY_MAJOR_VERSION >= 3
+/*
+ * Convert a C string in the PostgreSQL server encoding to a Python
+ * unicode object. Reference ownership is passed to the caller.
+ */
+static PyObject *
+PLyUnicode_FromString(const char *s)
+{
+ char *utf8string;
+ PyObject *o;
+
+ utf8string = (char *) pg_do_encoding_conversion((unsigned char *) s,
+ strlen(s),
+ GetDatabaseEncoding(),
+ PG_UTF8);
+
+ o = PyUnicode_FromString(utf8string);
+
+ if (utf8string != s)
+ pfree(utf8string);
+
+ return o;
+}
+#endif /* PY_MAJOR_VERSION >= 3 */
diff --git a/src/pl/plpython/sql/plpython_test.sql b/src/pl/plpython/sql/plpython_test.sql
index 161399f..7cae124 100644
--- a/src/pl/plpython/sql/plpython_test.sql
+++ b/src/pl/plpython/sql/plpython_test.sql
@@ -1,10 +1,15 @@
-- first some tests of basic functionality
+CREATE LANGUAGE plpython2u;
-- really stupid function just to get the module loaded
CREATE FUNCTION stupid() RETURNS text AS 'return "zarkon"' LANGUAGE plpythonu;
select stupid();
+-- check 2/3 versioning
+CREATE FUNCTION stupidn() RETURNS text AS 'return "zarkon"' LANGUAGE plpython2u;
+
+select stupidn();
-- test multiple arguments
CREATE FUNCTION argument_test_one(u users, a1 text, a2 text) RETURNS text
diff --git a/src/pl/plpython/sql/plpython_trigger.sql b/src/pl/plpython/sql/plpython_trigger.sql
index d6f441f..c60a673 100644
--- a/src/pl/plpython/sql/plpython_trigger.sql
+++ b/src/pl/plpython/sql/plpython_trigger.sql
@@ -67,7 +67,7 @@ SELECT * FROM users;
CREATE TABLE trigger_test
(i int, v text );
-CREATE FUNCTION trigger_data() returns trigger language plpythonu as $$
+CREATE FUNCTION trigger_data() RETURNS trigger LANGUAGE plpythonu AS $$
if 'relid' in TD:
TD['relid'] = "bogus:12345"