Support Python 3 tests under MSVC

Started by Andrew Dunstanover 7 years ago4 messages
#1Andrew Dunstan
andrew.dunstan@2ndquadrant.com
2 attachment(s)

Here are a couple of patches, one for master and one for the back
branches from 9.5 to 10 to allow testing of plpython3 and associated
contrib modules when building with MSVC.

I've tested this out on a Windows 2016 machine with Visual Studio 2017.

I'd like to apply these - they only affect the testing script
vcregress.pl, so they should be very low risk.

cheers

andrew

--
Andrew Dunstan https://www.2ndQuadrant.com
PostgreSQL Development, 24x7 Support, Remote DBA, Training & Services

Attachments:

plpython3.master.patchtext/x-patch; name=plpython3.master.patchDownload
diff --git a/src/tools/msvc/vcregress.pl b/src/tools/msvc/vcregress.pl
index f24f975..5dc72ff 100644
--- a/src/tools/msvc/vcregress.pl
+++ b/src/tools/msvc/vcregress.pl
@@ -246,6 +246,53 @@ sub taptest
 	exit $status if $status;
 }
 
+sub mangle_plpython3
+{
+	my $tests = shift;
+	mkdir "results" unless -d "results";
+	mkdir "sql/python3";
+	mkdir "results/python3";
+	mkdir "expected/python3";
+
+	foreach my $test (@$tests)
+	{
+		local $/ = undef;
+		foreach my $dir ('sql','expected')
+		{
+			my $extension = ($dir eq 'sql' ? 'sql' : 'out');
+
+			my @files = glob("$dir/$test.$extension $dir/${test}_[0-9].$extension");
+			foreach my $file (@files)
+			{
+				open(my $handle, "$file") || die "test file $file not found";
+				my $contents = <$handle>;
+				close($handle);
+				map
+				{
+					s/except ([[:alpha:]][[:alpha:].]*), *([[:alpha:]][[:alpha:]]*):/except $1 as $2:/g;
+					s/<type 'exceptions\.([[:alpha:]]*)'>/<class '$1'>/g;
+					s/<type 'long'>/<class 'int'>/g;
+					s/([0-9][0-9]*)L/$1/g;
+					s/([ [{])u"/$1"/g;
+					s/([ [{])u'/$1'/g;
+					s/def next/def __next__/g;
+					s/LANGUAGE plpython2?u/LANGUAGE plpython3u/g;
+					s/EXTENSION ([^ ]*_)*plpython2?u/EXTENSION $1plpython3u/g;
+					s/installing required extension "plpython2u"/installing required extension "plpython3u"/g;
+				} $contents;
+				my $base = basename $file;
+				open($handle, ">$dir/python3/$base") || die "opening python 3 file for $file";
+				print $handle $contents;
+				close($handle);
+			}
+		}
+	}
+	map { $_ =~ s!^!python3/!; } @$tests;
+	return @$tests;
+}
+
+
+
 sub plcheck
 {
 	chdir "$topdir/src/pl";
@@ -268,7 +315,8 @@ sub plcheck
 		}
 		if ($lang eq 'plpython')
 		{
-			next unless -d "$topdir/$Config/plpython2";
+			next unless -d "$topdir/$Config/plpython2" ||
+				-d "$topdir/$Config/plpython3";
 			$lang = 'plpythonu';
 		}
 		else
@@ -278,6 +326,8 @@ sub plcheck
 		my @lang_args = ("--load-extension=$lang");
 		chdir $dir;
 		my @tests = fetchTests();
+		@tests = mangle_plpython3(\@tests)
+			if $lang eq 'plpythonu' && -d "$topdir/$Config/plpython3";
 		if ($lang eq 'plperl')
 		{
 
@@ -293,6 +343,10 @@ sub plcheck
 				push(@tests, 'plperl_plperlu');
 			}
 		}
+		elsif ($lang eq 'plpythonu' && -d "$topdir/$Config/plpython3")
+		{
+			@lang_args = ();
+		}
 		print
 		  "============================================================\n";
 		print "Checking $lang\n";
@@ -335,6 +389,8 @@ sub subdircheck
 		die "Python not enabled in configuration"
 		  if !defined($config->{python});
 
+		@opts = grep { $_ !~ /plpythonu/ } @opts;
+
 		# Attempt to get python version and location.
 		# Assume python.exe in specified dir.
 		my $pythonprog = "import sys;" . "print(str(sys.version_info[0]))";
@@ -349,10 +405,7 @@ sub subdircheck
 		}
 		else
 		{
-
-			# disable tests on python3 for now.
-			chdir "..";
-			return;
+			@tests = mangle_plpython3(\@tests);
 		}
 	}
 
@@ -363,6 +416,7 @@ sub subdircheck
 		"$topdir/$Config/pg_regress/pg_regress",
 		"--bindir=${topdir}/${Config}/psql",
 		"--dbname=contrib_regression", @opts, @tests);
+	print join(' ',@args),"\n";
 	system(@args);
 	chdir "..";
 }
@@ -378,11 +432,8 @@ sub contribcheck
 		next if ($module eq "uuid-ossp"     && !defined($config->{uuid}));
 		next if ($module eq "sslinfo"       && !defined($config->{openssl}));
 		next if ($module eq "xml2"          && !defined($config->{xml}));
-		next if ($module eq "hstore_plperl" && !defined($config->{perl}));
-		next if ($module eq "jsonb_plperl"  && !defined($config->{perl}));
-		next if ($module eq "hstore_plpython" && !defined($config->{python}));
-		next if ($module eq "jsonb_plpython"  && !defined($config->{python}));
-		next if ($module eq "ltree_plpython"  && !defined($config->{python}));
+		next if ($module =~ /_plperl$/      && !defined($config->{perl}));
+		next if ($module =~ /_plpython$/    && !defined($config->{python}));
 		next if ($module eq "sepgsql");
 
 		subdircheck("$topdir/contrib", $module);
plpython3.9_5-10.patchtext/x-patch; name=plpython3.9_5-10.patchDownload
diff --git a/src/tools/msvc/vcregress.pl b/src/tools/msvc/vcregress.pl
index 2904679..43d5d76 100644
--- a/src/tools/msvc/vcregress.pl
+++ b/src/tools/msvc/vcregress.pl
@@ -244,6 +244,53 @@ sub taptest
 	exit $status if $status;
 }
 
+sub mangle_plpython3
+{
+	my $tests = shift;
+	mkdir "results" unless -d "results";
+	mkdir "sql/python3";
+	mkdir "results/python3";
+	mkdir "expected/python3";
+
+	foreach my $test (@$tests)
+	{
+		local $/ = undef;
+		foreach my $dir ('sql','expected')
+		{
+			my $extension = ($dir eq 'sql' ? 'sql' : 'out');
+
+			my @files = glob("$dir/$test.$extension $dir/${test}_[0-9].$extension");
+			foreach my $file (@files)
+			{
+				open(my $handle, "$file") || die "test file $file not found";
+				my $contents = <$handle>;
+				close($handle);
+				map
+				{
+					s/except ([[:alpha:]][[:alpha:].]*), *([[:alpha:]][[:alpha:]]*):/except $1 as $2:/g;
+					s/<type 'exceptions\.([[:alpha:]]*)'>/<class '$1'>/g;
+					s/<type 'long'>/<class 'int'>/g;
+					s/([0-9][0-9]*)L/$1/g;
+					s/([ [{])u"/$1"/g;
+					s/([ [{])u'/$1'/g;
+					s/def next/def __next__/g;
+					s/LANGUAGE plpython2?u/LANGUAGE plpython3u/g;
+					s/EXTENSION ([^ ]*_)*plpython2?u/EXTENSION $1plpython3u/g;
+					s/installing required extension "plpython2u"/installing required extension "plpython3u"/g;
+				} $contents;
+				my $base = basename $file;
+				open($handle, ">$dir/python3/$base") || die "opening python 3 file for $file";
+				print $handle $contents;
+				close($handle);
+			}
+		}
+	}
+	map { $_ =~ s!^!python3/!; } @$tests;
+	return @$tests;
+}
+
+
+
 sub plcheck
 {
 	chdir "../../pl";
@@ -254,7 +301,8 @@ sub plcheck
 		my $lang = $pl eq 'tcl' ? 'pltcl' : $pl;
 		if ($lang eq 'plpython')
 		{
-			next unless -d "../../$Config/plpython2";
+			next unless -d "$topdir/$Config/plpython2" ||
+				-d "$topdir/$Config/plpython3";
 			$lang = 'plpythonu';
 		}
 		else
@@ -264,6 +312,8 @@ sub plcheck
 		my @lang_args = ("--load-extension=$lang");
 		chdir $pl;
 		my @tests = fetchTests();
+		@tests = mangle_plpython3(\@tests)
+			if $lang eq 'plpythonu' && -d "$topdir/$Config/plpython3";
 		if ($lang eq 'plperl')
 		{
 
@@ -279,6 +329,10 @@ sub plcheck
 				push(@tests, 'plperl_plperlu');
 			}
 		}
+		elsif ($lang eq 'plpythonu' && -d "$topdir/$Config/plpython3")
+		{
+			@lang_args = ();
+		}
 		print
 		  "============================================================\n";
 		print "Checking $lang\n";
@@ -320,6 +374,8 @@ sub subdircheck
 		die "Python not enabled in configuration"
 		  if !defined($config->{python});
 
+		@opts = grep { $_ !~ /plpythonu/ } @opts;
+
 		# Attempt to get python version and location.
 		# Assume python.exe in specified dir.
 		my $pythonprog = "import sys;" . "print(str(sys.version_info[0]))";
@@ -334,10 +390,7 @@ sub subdircheck
 		}
 		else
 		{
-
-			# disable tests on python3 for now.
-			chdir "..";
-			return;
+			@tests = mangle_plpython3(\@tests);
 		}
 	}
 
@@ -348,6 +401,7 @@ sub subdircheck
 		"$topdir/$Config/pg_regress/pg_regress",
 		"--bindir=${topdir}/${Config}/psql",
 		"--dbname=contrib_regression", @opts, @tests);
+	print join(' ',@args),"\n";
 	system(@args);
 	chdir "..";
 }
@@ -363,9 +417,8 @@ sub contribcheck
 		next if ($module eq "uuid-ossp"     && !defined($config->{uuid}));
 		next if ($module eq "sslinfo"       && !defined($config->{openssl}));
 		next if ($module eq "xml2"          && !defined($config->{xml}));
-		next if ($module eq "hstore_plperl" && !defined($config->{perl}));
-		next if ($module eq "hstore_plpython" && !defined($config->{python}));
-		next if ($module eq "ltree_plpython"  && !defined($config->{python}));
+		next if ($module =~ /_plperl$/      && !defined($config->{perl}));
+		next if ($module =~ /_plpython$/    && !defined($config->{python}));
 		next if ($module eq "sepgsql");
 
 		subdircheck("$topdir/contrib", $module);
#2Peter Eisentraut
peter.eisentraut@2ndquadrant.com
In reply to: Andrew Dunstan (#1)
Re: Support Python 3 tests under MSVC

On 4/30/18 15:52, Andrew Dunstan wrote:

I'd like to apply these - they only affect the testing script
vcregress.pl, so they should be very low risk.

In case there are concerns about maintaining a second copy of the
"mangle" script: The obvious answer is to rewrite that script in Python.
I tried that once upon a time but it got too complicated.

--
Peter Eisentraut http://www.2ndQuadrant.com/
PostgreSQL Development, 24x7 Support, Remote DBA, Training & Services

#3Yuriy Zhuravlev
stalkerg@gmail.com
In reply to: Andrew Dunstan (#1)
Re: Support Python 3 tests under MSVC

Interesting, this working fine more than year with CMake build version.

On Tue, 1 May 2018, 04:53 Andrew Dunstan, <andrew.dunstan@2ndquadrant.com>
wrote:

Show quoted text

Here are a couple of patches, one for master and one for the back
branches from 9.5 to 10 to allow testing of plpython3 and associated
contrib modules when building with MSVC.

I've tested this out on a Windows 2016 machine with Visual Studio 2017.

I'd like to apply these - they only affect the testing script
vcregress.pl, so they should be very low risk.

cheers

andrew

--
Andrew Dunstan https://www.2ndQuadrant.com
PostgreSQL Development, 24x7 Support, Remote DBA, Training & Services

#4Andrew Dunstan
andrew.dunstan@2ndquadrant.com
In reply to: Peter Eisentraut (#2)
1 attachment(s)
Re: Support Python 3 tests under MSVC

On Mon, Apr 30, 2018 at 4:52 PM, Peter Eisentraut
<peter.eisentraut@2ndquadrant.com> wrote:

On 4/30/18 15:52, Andrew Dunstan wrote:

I'd like to apply these - they only affect the testing script
vcregress.pl, so they should be very low risk.

In case there are concerns about maintaining a second copy of the
"mangle" script: The obvious answer is to rewrite that script in Python.
I tried that once upon a time but it got too complicated.

Well, as you can see I have rewritten it in perl :-) We could extract
that into a perl package and centralize it, but I'm not sure it's
worth the trouble.

Here's a new version of the master patch. I wasn't able to resist the
temptation to do a little cleaning up. In particular, we really don't
need to call python to tell us which version we have - we can do what
we do elsewhere and look to see what python build directory we have. I
also removed the completely redundant first parameter of
subdircheck().

cheers

andrew

--
Andrew Dunstan https://www.2ndQuadrant.com
PostgreSQL Development, 24x7 Support, Remote DBA, Training & Services

Attachments:

plpython3.v3.patchapplication/octet-stream; name=plpython3.v3.patchDownload
diff --git a/src/tools/msvc/Install.pm b/src/tools/msvc/Install.pm
index 4cc1b57c1b..884c330cea 100644
--- a/src/tools/msvc/Install.pm
+++ b/src/tools/msvc/Install.pm
@@ -461,16 +461,12 @@ sub CopyContribFiles
 		opendir($D, $subdir) || croak "Could not opendir on $subdir!\n";
 		while (my $d = readdir($D))
 		{
-
 			# These configuration-based exclusions must match vcregress.pl
 			next if ($d eq "uuid-ossp"       && !defined($config->{uuid}));
 			next if ($d eq "sslinfo"         && !defined($config->{openssl}));
 			next if ($d eq "xml2"            && !defined($config->{xml}));
-			next if ($d eq "hstore_plperl"   && !defined($config->{perl}));
-			next if ($d eq "jsonb_plperl"    && !defined($config->{perl}));
-			next if ($d eq "hstore_plpython" && !defined($config->{python}));
-			next if ($d eq "jsonb_plpython"  && !defined($config->{python}));
-			next if ($d eq "ltree_plpython"  && !defined($config->{python}));
+			next if ($d =~ /_plperl$/        && !defined($config->{perl}));
+			next if ($d =~ /_plpython$/      && !defined($config->{python}));
 			next if ($d eq "sepgsql");
 
 			CopySubdirFiles($subdir, $d, $config, $target);
diff --git a/src/tools/msvc/vcregress.pl b/src/tools/msvc/vcregress.pl
index f24f975e18..2b8341f4fc 100644
--- a/src/tools/msvc/vcregress.pl
+++ b/src/tools/msvc/vcregress.pl
@@ -246,6 +246,51 @@ sub taptest
 	exit $status if $status;
 }
 
+sub mangle_plpython3
+{
+	my $tests = shift;
+	mkdir "results" unless -d "results";
+	mkdir "sql/python3";
+	mkdir "results/python3";
+	mkdir "expected/python3";
+
+	foreach my $test (@$tests)
+	{
+		local $/ = undef;
+		foreach my $dir ('sql','expected')
+		{
+			my $extension = ($dir eq 'sql' ? 'sql' : 'out');
+
+			my @files = glob("$dir/$test.$extension $dir/${test}_[0-9].$extension");
+			foreach my $file (@files)
+			{
+				open(my $handle, "$file") || die "test file $file not found";
+				my $contents = <$handle>;
+				close($handle);
+				map
+				{
+					s/except ([[:alpha:]][[:alpha:].]*), *([[:alpha:]][[:alpha:]]*):/except $1 as $2:/g;
+					s/<type 'exceptions\.([[:alpha:]]*)'>/<class '$1'>/g;
+					s/<type 'long'>/<class 'int'>/g;
+					s/([0-9][0-9]*)L/$1/g;
+					s/([ [{])u"/$1"/g;
+					s/([ [{])u'/$1'/g;
+					s/def next/def __next__/g;
+					s/LANGUAGE plpython2?u/LANGUAGE plpython3u/g;
+					s/EXTENSION ([^ ]*_)*plpython2?u/EXTENSION $1plpython3u/g;
+					s/installing required extension "plpython2u"/installing required extension "plpython3u"/g;
+				} $contents;
+				my $base = basename $file;
+				open($handle, ">$dir/python3/$base") || die "opening python 3 file for $file";
+				print $handle $contents;
+				close($handle);
+			}
+		}
+	}
+	map { $_ =~ s!^!python3/!; } @$tests;
+	return @$tests;
+}
+
 sub plcheck
 {
 	chdir "$topdir/src/pl";
@@ -268,7 +313,8 @@ sub plcheck
 		}
 		if ($lang eq 'plpython')
 		{
-			next unless -d "$topdir/$Config/plpython2";
+			next unless -d "$topdir/$Config/plpython2" ||
+				-d "$topdir/$Config/plpython3";
 			$lang = 'plpythonu';
 		}
 		else
@@ -278,6 +324,8 @@ sub plcheck
 		my @lang_args = ("--load-extension=$lang");
 		chdir $dir;
 		my @tests = fetchTests();
+		@tests = mangle_plpython3(\@tests)
+			if $lang eq 'plpythonu' && -d "$topdir/$Config/plpython3";
 		if ($lang eq 'plperl')
 		{
 
@@ -293,6 +341,10 @@ sub plcheck
 				push(@tests, 'plperl_plperlu');
 			}
 		}
+		elsif ($lang eq 'plpythonu' && -d "$topdir/$Config/plpython3")
+		{
+			@lang_args = ();
+		}
 		print
 		  "============================================================\n";
 		print "Checking $lang\n";
@@ -311,7 +363,6 @@ sub plcheck
 
 sub subdircheck
 {
-	my $subdir = shift;
 	my $module = shift;
 
 	if (   !-d "$module/sql"
@@ -325,44 +376,35 @@ sub subdircheck
 	my @tests = fetchTests();
 	my @opts  = fetchRegressOpts();
 
-	# Add some options for transform modules, see their respective
-	# Makefile for more details regarding Python-version specific
+	# Special processing for python transform modules, see their respective
+	# Makefiles for more details regarding Python-version specific
 	# dependencies.
-	if (   $module eq "hstore_plpython"
-		|| $module eq "jsonb_plpython"
-		|| $module eq "ltree_plpython")
+	if ( $module =~ /_plpython$/ )
 	{
 		die "Python not enabled in configuration"
 		  if !defined($config->{python});
 
-		# Attempt to get python version and location.
-		# Assume python.exe in specified dir.
-		my $pythonprog = "import sys;" . "print(str(sys.version_info[0]))";
-		my $prefixcmd  = $config->{python} . "\\python -c \"$pythonprog\"";
-		my $pyver      = `$prefixcmd`;
-		die "Could not query for python version!\n" if $?;
-		chomp($pyver);
-		if ($pyver eq "2")
+		@opts = grep { $_ !~ /plpythonu/ } @opts;
+
+		if (-d "$topdir/$Config/plpython2")
 		{
 			push @opts, "--load-extension=plpythonu";
 			push @opts, '--load-extension=' . $module . 'u';
 		}
 		else
 		{
-
-			# disable tests on python3 for now.
-			chdir "..";
-			return;
+			# must be python 3
+			@tests = mangle_plpython3(\@tests);
 		}
 	}
 
-
 	print "============================================================\n";
 	print "Checking $module\n";
 	my @args = (
 		"$topdir/$Config/pg_regress/pg_regress",
 		"--bindir=${topdir}/${Config}/psql",
 		"--dbname=contrib_regression", @opts, @tests);
+	print join(' ',@args),"\n";
 	system(@args);
 	chdir "..";
 }
@@ -373,19 +415,15 @@ sub contribcheck
 	my $mstat = 0;
 	foreach my $module (glob("*"))
 	{
-
 		# these configuration-based exclusions must match Install.pm
 		next if ($module eq "uuid-ossp"     && !defined($config->{uuid}));
 		next if ($module eq "sslinfo"       && !defined($config->{openssl}));
 		next if ($module eq "xml2"          && !defined($config->{xml}));
-		next if ($module eq "hstore_plperl" && !defined($config->{perl}));
-		next if ($module eq "jsonb_plperl"  && !defined($config->{perl}));
-		next if ($module eq "hstore_plpython" && !defined($config->{python}));
-		next if ($module eq "jsonb_plpython"  && !defined($config->{python}));
-		next if ($module eq "ltree_plpython"  && !defined($config->{python}));
+		next if ($module =~ /_plperl$/      && !defined($config->{perl}));
+		next if ($module =~ /_plpython$/    && !defined($config->{python}));
 		next if ($module eq "sepgsql");
 
-		subdircheck("$topdir/contrib", $module);
+		subdircheck($module);
 		my $status = $? >> 8;
 		$mstat ||= $status;
 	}
@@ -398,7 +436,7 @@ sub modulescheck
 	my $mstat = 0;
 	foreach my $module (glob("*"))
 	{
-		subdircheck("$topdir/src/test/modules", $module);
+		subdircheck($module);
 		my $status = $? >> 8;
 		$mstat ||= $status;
 	}