Open 6.4 items

Started by Bruce Momjianabout 27 years ago39 messages
#1Bruce Momjian
maillist@candle.pha.pa.us

Additions
---------
CREATE TABLE test (x text, s serial) fails if no database creation permission
regression test all platforms

Serious Items
------------
change pg args for platforms that don't support argv changes
(setproctitle()?, sendmail hack?)

Docs
----
generate html/postscript documentation
(User's Guide, Administrator's Guide, Programmer's Guide) (Thomas)
make sure all changes are documented properly
markup and merge JDBC docs from Peter (Thomas, others??)
merge the release notes into doc/src/sgml/release.sgml (Bruce, Thomas)
generate new text-format INSTALL and README from sgml sources (Thomas)
markup of Jan's PL docs

Minor items
-----------
cnf-ify still can exhaust memory, make SET KSQO more generic
permissions on indexes: what do they do? should it be prevented?
allow multiple generic operators in expressions without the use of parentheses
document/trigger/rule so changes to pg_shadow create pg_pwd
large objects orphanage
improve group handling
improve PRIMARY KEY handling
generate postmaster pid file and remove flock/fcntl lock code
add ability to specifiy location of lock/socket files

-- 
  Bruce Momjian                        |  http://www.op.net/~candle
  maillist@candle.pha.pa.us            |  (610) 853-3000
  +  If your life is a hard drive,     |  830 Blythe Avenue
  +  Christ can be your backup.        |  Drexel Hill, Pennsylvania 19026
#2Noname
jwieck@debis.com
In reply to: Bruce Momjian (#1)
Re: [HACKERS] Open 6.4 items

Bruce Momjian wrote:

Additions
---------
CREATE TABLE test (x text, s serial) fails if no database creation permission
regression test all platforms

The patch below arranges that the sequence(s) get created
first. Unprivileged user now can create table with serial
columns.

Regression tested.

Jan

--

#======================================================================#
# It's easier to get forgiveness for being wrong than for being right. #
# Let's break this rule - forgive me. #
#======================================== jwieck@debis.com (Jan Wieck) #

*** analyze.c.orig	Wed Oct 28 14:50:47 1998
--- analyze.c	Wed Oct 28 14:59:12 1998
***************
*** 42,48 ****
  static Query *transformCursorStmt(ParseState *pstate, SelectStmt *stmt);
  static Query *transformCreateStmt(ParseState *pstate, CreateStmt *stmt);

! List *extras = NIL;

  /*
   * parse_analyze -
--- 42,49 ----
  static Query *transformCursorStmt(ParseState *pstate, SelectStmt *stmt);
  static Query *transformCreateStmt(ParseState *pstate, CreateStmt *stmt);

! List *extras_before = NIL;
! List *extras_after = NIL;

  /*
   * parse_analyze -
***************
*** 57,62 ****
--- 58,64 ----
  {
  	QueryTreeList *result;
  	ParseState *pstate;
+ 	Query *parsetree;
  	int			i = 0;

result = malloc(sizeof(QueryTreeList));
***************
*** 66,88 ****
while (pl != NIL)
{
pstate = make_parsestate(parentParseState);
! result->qtrees[i++] = transformStmt(pstate, lfirst(pl));
if (pstate->p_target_relation != NULL)
heap_close(pstate->p_target_relation);

! 		if (extras != NIL)
  		{
! 			result->len += length(extras);
  			result->qtrees = (Query **) realloc(result->qtrees, result->len * sizeof(Query *));
! 			while (extras != NIL)
  			{
! 				result->qtrees[i++] = transformStmt(pstate, lfirst(extras));
  				if (pstate->p_target_relation != NULL)
  					heap_close(pstate->p_target_relation);
! 				extras = lnext(extras);
  			}
  		}
! 		extras = NIL;
  		pl = lnext(pl);
  		pfree(pstate);
  	}
--- 68,107 ----
  	while (pl != NIL)
  	{
  		pstate = make_parsestate(parentParseState);
! 		parsetree = transformStmt(pstate, lfirst(pl));
  		if (pstate->p_target_relation != NULL)
  			heap_close(pstate->p_target_relation);
! 		if (extras_before != NIL)
  		{
! 			result->len += length(extras_before);
  			result->qtrees = (Query **) realloc(result->qtrees, result->len * sizeof(Query *));
! 			while (extras_before != NIL)
  			{
! 				result->qtrees[i++] = transformStmt(pstate, lfirst(extras_before));
  				if (pstate->p_target_relation != NULL)
  					heap_close(pstate->p_target_relation);
! 				extras_before = lnext(extras_before);
  			}
  		}
! 		extras_before = NIL;
! 
! 		result->qtrees[i++] = parsetree;
! 
! 		if (extras_after != NIL)
! 		{
! 			result->len += length(extras_after);
! 			result->qtrees = (Query **) realloc(result->qtrees, result->len * sizeof(Query *));
! 			while (extras_after != NIL)
! 			{
! 				result->qtrees[i++] = transformStmt(pstate, lfirst(extras_after));
! 				if (pstate->p_target_relation != NULL)
! 					heap_close(pstate->p_target_relation);
! 				extras_after = lnext(extras_after);
! 			}
! 		}
! 		extras_after = NIL;
! 
  		pl = lnext(pl);
  		pfree(pstate);
  	}
***************
*** 487,492 ****
--- 506,512 ----
  	Constraint *constraint;
  	List	   *keys;
  	Ident	   *key;
+ 	List	   *blist = NIL;
  	List	   *ilist = NIL;
  	IndexStmt  *index;
  	IndexElem  *iparam;
***************
*** 553,559 ****
  					elog(NOTICE, "CREATE TABLE will create implicit sequence %s for SERIAL column %s.%s",
  					  sequence->seqname, stmt->relname, column->colname);

! ilist = lcons(sequence, NIL);
}

  				if (column->constraints != NIL)
--- 573,579 ----
  					elog(NOTICE, "CREATE TABLE will create implicit sequence %s for SERIAL column %s.%s",
  					  sequence->seqname, stmt->relname, column->colname);

! blist = lcons(sequence, NIL);
}

if (column->constraints != NIL)
***************
*** 745,751 ****
}

q->utilityStmt = (Node *) stmt;
! extras = ilist;

  	return q;
  }
--- 765,772 ----
  	}

q->utilityStmt = (Node *) stmt;
! extras_before = blist;
! extras_after = ilist;

return q;
}

#3The Hermit Hacker
scrappy@hub.org
In reply to: Noname (#2)
Re: [HACKERS] Open 6.4 items

On Wed, 28 Oct 1998, Jan Wieck wrote:

Bruce Momjian wrote:

Additions
---------
CREATE TABLE test (x text, s serial) fails if no database creation permission
regression test all platforms

The patch below arranges that the sequence(s) get created
first. Unprivileged user now can create table with serial
columns.

Is this safe to leave until post-v6.4 released?

Regression tested.

Jan

--

#======================================================================#
# It's easier to get forgiveness for being wrong than for being right. #
# Let's break this rule - forgive me. #
#======================================== jwieck@debis.com (Jan Wieck) #

*** analyze.c.orig	Wed Oct 28 14:50:47 1998
--- analyze.c	Wed Oct 28 14:59:12 1998
***************
*** 42,48 ****
static Query *transformCursorStmt(ParseState *pstate, SelectStmt *stmt);
static Query *transformCreateStmt(ParseState *pstate, CreateStmt *stmt);

! List *extras = NIL;

/*
* parse_analyze -
--- 42,49 ----
static Query *transformCursorStmt(ParseState *pstate, SelectStmt *stmt);
static Query *transformCreateStmt(ParseState *pstate, CreateStmt *stmt);

! List *extras_before = NIL;
! List *extras_after = NIL;

/*
* parse_analyze -
***************
*** 57,62 ****
--- 58,64 ----
{
QueryTreeList *result;
ParseState *pstate;
+ 	Query *parsetree;
int			i = 0;

result = malloc(sizeof(QueryTreeList));
***************
*** 66,88 ****
while (pl != NIL)
{
pstate = make_parsestate(parentParseState);
! result->qtrees[i++] = transformStmt(pstate, lfirst(pl));
if (pstate->p_target_relation != NULL)
heap_close(pstate->p_target_relation);

! 		if (extras != NIL)
{
! 			result->len += length(extras);
result->qtrees = (Query **) realloc(result->qtrees, result->len * sizeof(Query *));
! 			while (extras != NIL)
{
! 				result->qtrees[i++] = transformStmt(pstate, lfirst(extras));
if (pstate->p_target_relation != NULL)
heap_close(pstate->p_target_relation);
! 				extras = lnext(extras);
}
}
! 		extras = NIL;
pl = lnext(pl);
pfree(pstate);
}
--- 68,107 ----
while (pl != NIL)
{
pstate = make_parsestate(parentParseState);
! 		parsetree = transformStmt(pstate, lfirst(pl));
if (pstate->p_target_relation != NULL)
heap_close(pstate->p_target_relation);
! 		if (extras_before != NIL)
{
! 			result->len += length(extras_before);
result->qtrees = (Query **) realloc(result->qtrees, result->len * sizeof(Query *));
! 			while (extras_before != NIL)
{
! 				result->qtrees[i++] = transformStmt(pstate, lfirst(extras_before));
if (pstate->p_target_relation != NULL)
heap_close(pstate->p_target_relation);
! 				extras_before = lnext(extras_before);
}
}
! 		extras_before = NIL;
! 
! 		result->qtrees[i++] = parsetree;
! 
! 		if (extras_after != NIL)
! 		{
! 			result->len += length(extras_after);
! 			result->qtrees = (Query **) realloc(result->qtrees, result->len * sizeof(Query *));
! 			while (extras_after != NIL)
! 			{
! 				result->qtrees[i++] = transformStmt(pstate, lfirst(extras_after));
! 				if (pstate->p_target_relation != NULL)
! 					heap_close(pstate->p_target_relation);
! 				extras_after = lnext(extras_after);
! 			}
! 		}
! 		extras_after = NIL;
! 
pl = lnext(pl);
pfree(pstate);
}
***************
*** 487,492 ****
--- 506,512 ----
Constraint *constraint;
List	   *keys;
Ident	   *key;
+ 	List	   *blist = NIL;
List	   *ilist = NIL;
IndexStmt  *index;
IndexElem  *iparam;
***************
*** 553,559 ****
elog(NOTICE, "CREATE TABLE will create implicit sequence %s for SERIAL column %s.%s",
sequence->seqname, stmt->relname, column->colname);

! ilist = lcons(sequence, NIL);
}

if (column->constraints != NIL)
--- 573,579 ----
elog(NOTICE, "CREATE TABLE will create implicit sequence %s for SERIAL column %s.%s",
sequence->seqname, stmt->relname, column->colname);

! blist = lcons(sequence, NIL);
}

if (column->constraints != NIL)
***************
*** 745,751 ****
}

q->utilityStmt = (Node *) stmt;
! extras = ilist;

return q;
}
--- 765,772 ----
}

q->utilityStmt = (Node *) stmt;
! extras_before = blist;
! extras_after = ilist;

return q;
}

Marc G. Fournier
Systems Administrator @ hub.org
primary: scrappy@hub.org secondary: scrappy@{freebsd|postgresql}.org

#4Bruce Momjian
maillist@candle.pha.pa.us
In reply to: Noname (#2)
Re: [HACKERS] Open 6.4 items

Thanks. Applied.

Bruce Momjian wrote:

Additions
---------
CREATE TABLE test (x text, s serial) fails if no database creation permission
regression test all platforms

The patch below arranges that the sequence(s) get created
first. Unprivileged user now can create table with serial
columns.

Regression tested.

Jan

--

#======================================================================#
# It's easier to get forgiveness for being wrong than for being right. #
# Let's break this rule - forgive me. #
#======================================== jwieck@debis.com (Jan Wieck) #

*** analyze.c.orig	Wed Oct 28 14:50:47 1998
--- analyze.c	Wed Oct 28 14:59:12 1998
***************
*** 42,48 ****
static Query *transformCursorStmt(ParseState *pstate, SelectStmt *stmt);
static Query *transformCreateStmt(ParseState *pstate, CreateStmt *stmt);

! List *extras = NIL;

/*
* parse_analyze -
--- 42,49 ----
static Query *transformCursorStmt(ParseState *pstate, SelectStmt *stmt);
static Query *transformCreateStmt(ParseState *pstate, CreateStmt *stmt);

! List *extras_before = NIL;
! List *extras_after = NIL;

/*
* parse_analyze -
***************
*** 57,62 ****
--- 58,64 ----
{
QueryTreeList *result;
ParseState *pstate;
+ 	Query *parsetree;
int			i = 0;

result = malloc(sizeof(QueryTreeList));
***************
*** 66,88 ****
while (pl != NIL)
{
pstate = make_parsestate(parentParseState);
! result->qtrees[i++] = transformStmt(pstate, lfirst(pl));
if (pstate->p_target_relation != NULL)
heap_close(pstate->p_target_relation);

! 		if (extras != NIL)
{
! 			result->len += length(extras);
result->qtrees = (Query **) realloc(result->qtrees, result->len * sizeof(Query *));
! 			while (extras != NIL)
{
! 				result->qtrees[i++] = transformStmt(pstate, lfirst(extras));
if (pstate->p_target_relation != NULL)
heap_close(pstate->p_target_relation);
! 				extras = lnext(extras);
}
}
! 		extras = NIL;
pl = lnext(pl);
pfree(pstate);
}
--- 68,107 ----
while (pl != NIL)
{
pstate = make_parsestate(parentParseState);
! 		parsetree = transformStmt(pstate, lfirst(pl));
if (pstate->p_target_relation != NULL)
heap_close(pstate->p_target_relation);
! 		if (extras_before != NIL)
{
! 			result->len += length(extras_before);
result->qtrees = (Query **) realloc(result->qtrees, result->len * sizeof(Query *));
! 			while (extras_before != NIL)
{
! 				result->qtrees[i++] = transformStmt(pstate, lfirst(extras_before));
if (pstate->p_target_relation != NULL)
heap_close(pstate->p_target_relation);
! 				extras_before = lnext(extras_before);
}
}
! 		extras_before = NIL;
! 
! 		result->qtrees[i++] = parsetree;
! 
! 		if (extras_after != NIL)
! 		{
! 			result->len += length(extras_after);
! 			result->qtrees = (Query **) realloc(result->qtrees, result->len * sizeof(Query *));
! 			while (extras_after != NIL)
! 			{
! 				result->qtrees[i++] = transformStmt(pstate, lfirst(extras_after));
! 				if (pstate->p_target_relation != NULL)
! 					heap_close(pstate->p_target_relation);
! 				extras_after = lnext(extras_after);
! 			}
! 		}
! 		extras_after = NIL;
! 
pl = lnext(pl);
pfree(pstate);
}
***************
*** 487,492 ****
--- 506,512 ----
Constraint *constraint;
List	   *keys;
Ident	   *key;
+ 	List	   *blist = NIL;
List	   *ilist = NIL;
IndexStmt  *index;
IndexElem  *iparam;
***************
*** 553,559 ****
elog(NOTICE, "CREATE TABLE will create implicit sequence %s for SERIAL column %s.%s",
sequence->seqname, stmt->relname, column->colname);

! ilist = lcons(sequence, NIL);
}

if (column->constraints != NIL)
--- 573,579 ----
elog(NOTICE, "CREATE TABLE will create implicit sequence %s for SERIAL column %s.%s",
sequence->seqname, stmt->relname, column->colname);

! blist = lcons(sequence, NIL);
}

if (column->constraints != NIL)
***************
*** 745,751 ****
}

q->utilityStmt = (Node *) stmt;
! extras = ilist;

return q;
}
--- 765,772 ----
}

q->utilityStmt = (Node *) stmt;
! extras_before = blist;
! extras_after = ilist;

return q;
}

-- 
  Bruce Momjian                        |  http://www.op.net/~candle
  maillist@candle.pha.pa.us            |  (610) 853-3000
  +  If your life is a hard drive,     |  830 Blythe Avenue
  +  Christ can be your backup.        |  Drexel Hill, Pennsylvania 19026
#5Bruce Momjian
maillist@candle.pha.pa.us
In reply to: Bruce Momjian (#4)
Re: [HACKERS] Open 6.4 items

Marc wrote:

On Wed, 28 Oct 1998, Jan Wieck wrote:

Bruce Momjian wrote:

Additions
---------
CREATE TABLE test (x text, s serial) fails if no database creation permission
regression test all platforms

The patch below arranges that the sequence(s) get created
first. Unprivileged user now can create table with serial
columns.

Is this safe to leave until post-v6.4 released?

I just did it now to let Bruce get rid of the open item. If
it shouldn't be in 6.4, why is it on it's open item list
then?

It has been on the open items. In fact, it is the only hot item, except
for the 'pg args' thing, which is not as hot.

It is serious breakage, in the sense that if we don't fix it, we must
remove reference to the new SERIAL type.

As to why no one fixed it earlier, ...? Must be the same reason 'ps
args' is not fixed. :-)

-- 
  Bruce Momjian                        |  http://www.op.net/~candle
  maillist@candle.pha.pa.us            |  (610) 853-3000
  +  If your life is a hard drive,     |  830 Blythe Avenue
  +  Christ can be your backup.        |  Drexel Hill, Pennsylvania 19026
#6Noname
jwieck@debis.com
In reply to: The Hermit Hacker (#3)
Re: [HACKERS] Open 6.4 items

Marc wrote:

On Wed, 28 Oct 1998, Jan Wieck wrote:

Bruce Momjian wrote:

Additions
---------
CREATE TABLE test (x text, s serial) fails if no database creation permission
regression test all platforms

The patch below arranges that the sequence(s) get created
first. Unprivileged user now can create table with serial
columns.

Is this safe to leave until post-v6.4 released?

I just did it now to let Bruce get rid of the open item. If
it shouldn't be in 6.4, why is it on it's open item list
then?

Jan

--

#======================================================================#
# It's easier to get forgiveness for being wrong than for being right. #
# Let's break this rule - forgive me. #
#======================================== jwieck@debis.com (Jan Wieck) #

#7Noname
jwieck@debis.com
In reply to: Bruce Momjian (#4)
Re: [HACKERS] Open 6.4 items

Thanks. Applied.

Bruce Momjian wrote:

Additions
---------
CREATE TABLE test (x text, s serial) fails if no database creation permission
regression test all platforms

The patch below arranges that the sequence(s) get created
first. Unprivileged user now can create table with serial
columns.

Regression tested.

So Marc's assumption that his BETA3 is what we release was
false. We really know how to trigger his hot buttons - eh?
:-)

Jan

--

#======================================================================#
# It's easier to get forgiveness for being wrong than for being right. #
# Let's break this rule - forgive me. #
#======================================== jwieck@debis.com (Jan Wieck) #

#8Thomas G. Lockhart
lockhart@alumni.caltech.edu
In reply to: Bruce Momjian (#5)
Re: [HACKERS] Open 6.4 items

It is serious breakage, in the sense that if we don't fix it, we must
remove reference to the new SERIAL type.
As to why no one fixed it earlier, ...? Must be the same reason 'ps
args' is not fixed. :-)

I'm comfortable with Jan's patches, and it's my code he's mucking with
:) For the transformation from serial to int4 with index and sequence,
the only time that code gets traversed is for the serial type. So it
shouldn't have hidden side effects (yeah, sure...)

'Course, I see Bruce has already committed it anyway...

- Tom

#9Bruce Momjian
maillist@candle.pha.pa.us
In reply to: Thomas G. Lockhart (#8)
Re: [HACKERS] Open 6.4 items

Here are the open items. Thanks to Jan, the only 'hot' item left is the
ps args issue. People on non-BSD platforms will see all their backends
called 'postmaster', because argv[0] changes do not reflect in ps arg
displays.

I have asked that at least we get set_proctitle() working for Linux, but
no one wants to do it, and we can't remove the it because we would have
to add the exec() back into backend creation, which is impossible at
this point.

I'm running Linux 2.1.88 and get

15572 p2 S 0:01 /usr/local/pgsql/bin/postmaster -o -F
16121 p2 S 0:01 /usr/local/pgsql/bin/postgres localhost twieck twieck idle

from ps. So what isn't working?

Whoh, this is a shock. I thought this trick did not work under Linux.
If it does, and no one has complained, we can consider the issue closed.

Massimo did a nice job of moving my ps status stuff into pg_status.h,
but it looks like the same code that modifies argv[0]. Perhaps it is
the Linux version you are using.

It give us a cheap 'db monitor' capability, wrapped into the 'ps'
command.

-- 
  Bruce Momjian                        |  http://www.op.net/~candle
  maillist@candle.pha.pa.us            |  (610) 853-3000
  +  If your life is a hard drive,     |  830 Blythe Avenue
  +  Christ can be your backup.        |  Drexel Hill, Pennsylvania 19026
#10Andreas Zeugswetter
andreas.zeugswetter@telecom.at
In reply to: Bruce Momjian (#9)
AW: [HACKERS] Open 6.4 items

I'm running Linux 2.1.88 and get

15572 p2 S 0:01 /usr/local/pgsql/bin/postmaster -o -F
16121 p2 S 0:01 /usr/local/pgsql/bin/postgres localhost twieck twieck idle

from ps. So what isn't working?

Also works on AIX :-)

#11Brook Milligan
brook@trillium.NMSU.Edu
In reply to: Bruce Momjian (#9)
Re: [HACKERS] Open 6.4 items

I'm running Linux 2.1.88 and get

15572 p2 S 0:01 /usr/local/pgsql/bin/postmaster -o -F
16121 p2 S 0:01 /usr/local/pgsql/bin/postgres localhost twieck twieck idle

from ps. So what isn't working?

Whoh, this is a shock. I thought this trick did not work under Linux.
If it does, and no one has complained, we can consider the issue closed.

For whatever it's worth, I see the same status stuff changing under
NetBSD 1.3.2. Seems to work fine.

Cheers,
Brook

#12Thomas G. Lockhart
lockhart@alumni.caltech.edu
In reply to: Noname (#7)
Re: [HACKERS] Open 6.4 items

So Marc's assumption that his BETA3 is what we release was
false. We really know how to trigger his hot buttons - eh?
:-)

Good point. I just committed a _very_ small bit of code in the ODBC
interface which is non-essential, but potentially useful.

Also, he will have to rebuild a tarball one more time anyway to get the
new docs, which won't be finished until the last minute.

Just to get his blood pressure up, I'll mention that there is a chance
I'll ask for a couple of extra days to finish docs, but I'm hoping to
have them done on time :)

- Tom

#13The Hermit Hacker
scrappy@hub.org
In reply to: Noname (#7)
Re: [HACKERS] Open 6.4 items

On Wed, 28 Oct 1998, Jan Wieck wrote:

Thanks. Applied.

Bruce Momjian wrote:

Additions
---------
CREATE TABLE test (x text, s serial) fails if no database creation permission
regression test all platforms

The patch below arranges that the sequence(s) get created
first. Unprivileged user now can create table with serial
columns.

Regression tested.

So Marc's assumption that his BETA3 is what we release was
false. We really know how to trigger his hot buttons - eh?
:-)

If it was 'the release', it wouldn't have been named BETA :) I'll
build a BETA4 tomorrow...release set for Monday...

Marc G. Fournier
Systems Administrator @ hub.org
primary: scrappy@hub.org secondary: scrappy@{freebsd|postgresql}.org

#14The Hermit Hacker
scrappy@hub.org
In reply to: Brook Milligan (#11)
Re: [HACKERS] Open 6.4 items

Doesn't work under Solaris, but, then again, neither does sendamil's, so
that isn't a shock :)

On Wed, 28 Oct 1998, Brook Milligan wrote:

I'm running Linux 2.1.88 and get

15572 p2 S 0:01 /usr/local/pgsql/bin/postmaster -o -F
16121 p2 S 0:01 /usr/local/pgsql/bin/postgres localhost twieck twieck idle

from ps. So what isn't working?

Whoh, this is a shock. I thought this trick did not work under Linux.
If it does, and no one has complained, we can consider the issue closed.

For whatever it's worth, I see the same status stuff changing under
NetBSD 1.3.2. Seems to work fine.

Cheers,
Brook

Marc G. Fournier
Systems Administrator @ hub.org
primary: scrappy@hub.org secondary: scrappy@{freebsd|postgresql}.org

#15Bruce Momjian
maillist@candle.pha.pa.us
In reply to: The Hermit Hacker (#14)
Re: [HACKERS] Open 6.4 items

Doesn't work under Solaris, but, then again, neither does sendamil's, so
that isn't a shock :)

Man, if sendmail's doesn't work, that is really broken.

-- 
  Bruce Momjian                        |  http://www.op.net/~candle
  maillist@candle.pha.pa.us            |  (610) 853-3000
  +  If your life is a hard drive,     |  830 Blythe Avenue
  +  Christ can be your backup.        |  Drexel Hill, Pennsylvania 19026
#16The Hermit Hacker
scrappy@hub.org
In reply to: Bruce Momjian (#15)
Re: [HACKERS] Open 6.4 items

On Wed, 28 Oct 1998, Bruce Momjian wrote:

Doesn't work under Solaris, but, then again, neither does sendamil's, so
that isn't a shock :)

Man, if sendmail's doesn't work, that is really broken.

Solaris just doesn't have any mechanisms to work around the
limitation, I guess *shrug* It really sucks when you want to SIGHUP the
"parent process", which, under FreeBSD at least, is the one that states:
-accepting connections, but under Solaris they are *all* the same :)

Marc G. Fournier
Systems Administrator @ hub.org
primary: scrappy@hub.org secondary: scrappy@{freebsd|postgresql}.org

#17Billy G. Allie
Bill.Allie@mug.org
In reply to: The Hermit Hacker (#14)
Re: [HACKERS] Open 6.4 items

Doesn't work under UnixWare, either.

The Hermit Hacker wrote:

Doesn't work under Solaris, but, then again, neither does sendamil's, so
that isn't a shock :)

On Wed, 28 Oct 1998, Brook Milligan wrote:

I'm running Linux 2.1.88 and get

15572 p2 S 0:01 /usr/local/pgsql/bin/postmaster -o -F
16121 p2 S 0:01 /usr/local/pgsql/bin/postgres localhost twieck twieck idle

from ps. So what isn't working?

Whoh, this is a shock. I thought this trick did not work under Linux.
If it does, and no one has complained, we can consider the issue closed.

For whatever it's worth, I see the same status stuff changing under
NetBSD 1.3.2. Seems to work fine.

Cheers,
Brook

Marc G. Fournier
Systems Administrator @ hub.org
primary: scrappy@hub.org secondary: scrappy@{freebsd|postgresql}.org

--
____ | Billy G. Allie | Domain....: Bill.Allie@mug.org
| /| | 7436 Hartwell | Compuserve: 76337,2061
|-/-|----- | Dearborn, MI 48126| MSN.......: B_G_Allie@email.msn.com
|/ |LLIE | (313) 582-1540 |

#18Billy G. Allie
Bill.Allie@mug.org
In reply to: Bruce Momjian (#15)
Re: [HACKERS] Open 6.4 items

Bruce Momjian wrote:

Doesn't work under Solaris, but, then again, neither does sendamil's, so
that isn't a shock :)

Man, if sendmail's doesn't work, that is really broken.

If you want strange, here is the output from the UnixWare 7 ps command with
some different options:

$ ps -ef | fgrep post
pgsql 10775 1 TS 80 0 01:34:12 ? 0:00 postmaster -S -i -o -F
pgsql 10869 10775 TS 80 0 01:45:30 ? 0:00 postmaster -S -i -o -F

$ ps -e | fgrep post
10775 TS 80 ? 0:00 postgres
10869 TS 80 ? 0:00 postgres

Notice that the PIDs are the same even if the program name is different
depending on the use of the 'f' option.
--
____ | Billy G. Allie | Domain....: Bill.Allie@mug.org
| /| | 7436 Hartwell | Compuserve: 76337,2061
|-/-|----- | Dearborn, MI 48126| MSN.......: B_G_Allie@email.msn.com
|/ |LLIE | (313) 582-1540 |

#19Bruce Momjian
maillist@candle.pha.pa.us
In reply to: Billy G. Allie (#18)
Re: [HACKERS] Open 6.4 items

Bruce Momjian wrote:

Doesn't work under Solaris, but, then again, neither does sendamil's, so
that isn't a shock :)

Man, if sendmail's doesn't work, that is really broken.

If you want strange, here is the output from the UnixWare 7 ps command with
some different options:

$ ps -ef | fgrep post
pgsql 10775 1 TS 80 0 01:34:12 ? 0:00 postmaster -S -i -o -F
pgsql 10869 10775 TS 80 0 01:45:30 ? 0:00 postmaster -S -i -o -F

$ ps -e | fgrep post
10775 TS 80 ? 0:00 postgres
10869 TS 80 ? 0:00 postgres

Notice that the PIDs are the same even if the program name is different
depending on the use of the 'f' option.

Wow, that really is strange. Seems like with the -f, it looks in one
place, as though it knows the child did not exec, so they params must be
the same, while with no -f, it looks in the proper area. The really
weird thing is that even the postmaster is called postgres. That is
bizarre.

-- 
  Bruce Momjian                        |  http://www.op.net/~candle
  maillist@candle.pha.pa.us            |  (610) 853-3000
  +  If your life is a hard drive,     |  830 Blythe Avenue
  +  Christ can be your backup.        |  Drexel Hill, Pennsylvania 19026
#20Bruce Momjian
maillist@candle.pha.pa.us
In reply to: Billy G. Allie (#18)
Re: [HACKERS] Open 6.4 items

Bruce Momjian wrote:

Doesn't work under Solaris, but, then again, neither does sendamil's, so
that isn't a shock :)

Man, if sendmail's doesn't work, that is really broken.

If you want strange, here is the output from the UnixWare 7 ps command with
some different options:

$ ps -ef | fgrep post
pgsql 10775 1 TS 80 0 01:34:12 ? 0:00 postmaster -S -i -o -F
pgsql 10869 10775 TS 80 0 01:45:30 ? 0:00 postmaster -S -i -o -F

$ ps -e | fgrep post
10775 TS 80 ? 0:00 postgres
10869 TS 80 ? 0:00 postgres

Notice that the PIDs are the same even if the program name is different
depending on the use of the 'f' option.

That is really bizarre. The postmaster name is changed to postgres too!

Ah, I now know why things work on Linux. Massimo did more than just
parameterize my argv code. In include/utils/ps_status.h, he put in
Linux-specific code to handle the ps args. Seems under Linux, only
argv[0] can be modified for ps display, for some reason. He erases the
other args, and puts everything in argv[0].

Under non-linux, he uses the argv[0-4] for various displays, like my
orignal code. The major issue is that for Linux, he writes directly
into argv[0] memory. He sets the normal i/o static parameters in the
start of the string, then marks the next position, and writes status
information into there.

In non-Linux, we don't write into argv[0], but change argv[0..4] to
point to our own alloc'ed strings, where the memory sits in our own
address space, not in the proc 'environment' address space. Getting
'ps' to display process address space strings is a real trick, so it is
not surprising some O/S's don't support it.

He may have done it all in argv[0] because he is expanding the size of
argv[0], and if they are layed out sequentially in environment memory,
the changes would over-write each other. What I don't understand is why
ps when it goes to look at argv[1], does not see garbage because it may
be pointing into the middle of the argv[0] string.) Can a Linux user
use ps to display all the params, like 'ps -ef' and see if it is showing
argv[1] as part of the argv[0] string? You may have to get a wide
display to see it all.

The great news is that it seems it is working to write directly into the
environment address space under Linux, so we don't need set_proctitle,
and it is very fast.

Non-linux platforms are not writing directly to environment memory, but
are using the argv[0..4] trick.

FYI, I have re-exec'ed the postmaster in postmaster.h so it has 4 args.
argc is passed as a value to the program, so you can't change it, and if
you don't do the re-exec, ps only looks at the argv[0], and under
non-linux, that is bad because we use argv[0...4].

-- 
  Bruce Momjian                        |  http://www.op.net/~candle
  maillist@candle.pha.pa.us            |  (610) 853-3000
  +  If your life is a hard drive,     |  830 Blythe Avenue
  +  Christ can be your backup.        |  Drexel Hill, Pennsylvania 19026
#21Matthew N. Dodd
winter@jurai.net
In reply to: The Hermit Hacker (#16)
Re: [HACKERS] Open 6.4 items

On Thu, 29 Oct 1998, The Hermit Hacker wrote:

Solaris just doesn't have any mechanisms to work around the
limitation, I guess *shrug* It really sucks when you want to SIGHUP
the "parent process", which, under FreeBSD at least, is the one that
states: -accepting connections, but under Solaris they are *all* the
same :)

$ ps -eaf
UID PID PPID C STIME TTY TIME CMD
root 0 0 0 Oct 12 ? 0:01 sched
root 1 0 0 Oct 12 ? 0:15 /etc/init -
...

You'll note the 'PPID' field.

3 guesses what that stands for.

--
| Matthew N. Dodd | 78 280Z | 75 164E | 84 245DL | FreeBSD/NetBSD/Sprite/VMS |
| winter@jurai.net | This Space For Rent | ix86,sparc,m68k,pmax,vax |
| http://www.jurai.net/~winter | Are you k-rad elite enough for my webpage? |

#22Noname
jwieck@debis.com
In reply to: Matthew N. Dodd (#21)
Re: [HACKERS] Open 6.4 items

On Thu, 29 Oct 1998, The Hermit Hacker wrote:

Solaris just doesn't have any mechanisms to work around the
limitation, I guess *shrug* It really sucks when you want to SIGHUP
the "parent process", which, under FreeBSD at least, is the one that
states: -accepting connections, but under Solaris they are *all* the
same :)

$ ps -eaf
UID PID PPID C STIME TTY TIME CMD
root 0 0 0 Oct 12 ? 0:01 sched
root 1 0 0 Oct 12 ? 0:15 /etc/init -
...

You'll note the 'PPID' field.

3 guesses what that stands for.

Don't see how this is related to the topic - sorry.

PPID is the parent process ID. sched has no parent (it's a
kernel pseudo process) and init has sched as father. For all
other processes the PPID is set to init's PID at the time
their father dies (you'll see lot's of PPID=1).

But this all has nothing to do with changing the CMD column
of the ps output from inside a running process.

Jan

--

#======================================================================#
# It's easier to get forgiveness for being wrong than for being right. #
# Let's break this rule - forgive me. #
#======================================== jwieck@debis.com (Jan Wieck) #

#23Matthew N. Dodd
winter@jurai.net
In reply to: Noname (#22)
Re: [HACKERS] Open 6.4 items

On Thu, 29 Oct 1998, Jan Wieck wrote:

On Thu, 29 Oct 1998, The Hermit Hacker wrote:

Solaris just doesn't have any mechanisms to work around the
limitation, I guess *shrug* It really sucks when you want to SIGHUP
the "parent process", which, under FreeBSD at least, is the one that
states: -accepting connections, but under Solaris they are *all* the
same :)

$ ps -eaf
UID PID PPID C STIME TTY TIME CMD
root 0 0 0 Oct 12 ? 0:01 sched
root 1 0 0 Oct 12 ? 0:15 /etc/init -
...

You'll note the 'PPID' field.

3 guesses what that stands for.

Don't see how this is related to the topic - sorry.

I really have to start explaining things using more words don't I.

PPID is the parent process ID. sched has no parent (it's a
kernel pseudo process) and init has sched as father. For all
other processes the PPID is set to init's PID at the time
their father dies (you'll see lot's of PPID=1).

Reread the bit of text I quoted. Read my reply. How does my reply
address the problem scrappy had?

The bits of ps output I quoted were only serving to demonstrate actual
data produced by the ps command I used. The actual commands weren't
important, only the PID and PPID fields were.

But this all has nothing to do with changing the CMD column
of the ps output from inside a running process.

No, but changing the CMD column is only eye-candy. Its not necessary to
be able to tell which processes are children and which is the parent.

--
| Matthew N. Dodd | 78 280Z | 75 164E | 84 245DL | FreeBSD/NetBSD/Sprite/VMS |
| winter@jurai.net | This Space For Rent | ix86,sparc,m68k,pmax,vax |
| http://www.jurai.net/~winter | Are you k-rad elite enough for my webpage? |

#24Taral
taral@mail.utexas.edu
In reply to: Bruce Momjian (#19)
RE: [HACKERS] Open 6.4 items

Wow, that really is strange. Seems like with the -f, it looks in one
place, as though it knows the child did not exec, so they params must be
the same, while with no -f, it looks in the proper area. The really
weird thing is that even the postmaster is called postgres. That is
bizarre.

Yes. With -f, it looks in the process's argv... without, it looks in the
task struct where only the executable name was stored (no path or anything
else). It's a feature to prevent processes that spoof as something else...
(not very effective, IMHO)

Taral

#25The Hermit Hacker
scrappy@hub.org
In reply to: Billy G. Allie (#17)
Re: [HACKERS] Open 6.4 items

On Thu, 29 Oct 1998, Billy G. Allie wrote:

Doesn't work under UnixWare, either.

How about sendmail? That is "broken" too?

The Hermit Hacker wrote:

Doesn't work under Solaris, but, then again, neither does sendamil's, so
that isn't a shock :)

On Wed, 28 Oct 1998, Brook Milligan wrote:

I'm running Linux 2.1.88 and get

15572 p2 S 0:01 /usr/local/pgsql/bin/postmaster -o -F
16121 p2 S 0:01 /usr/local/pgsql/bin/postgres localhost twieck twieck idle

from ps. So what isn't working?

Whoh, this is a shock. I thought this trick did not work under Linux.
If it does, and no one has complained, we can consider the issue closed.

For whatever it's worth, I see the same status stuff changing under
NetBSD 1.3.2. Seems to work fine.

Cheers,
Brook

Marc G. Fournier
Systems Administrator @ hub.org
primary: scrappy@hub.org secondary: scrappy@{freebsd|postgresql}.org

--
____ | Billy G. Allie | Domain....: Bill.Allie@mug.org
| /| | 7436 Hartwell | Compuserve: 76337,2061
|-/-|----- | Dearborn, MI 48126| MSN.......: B_G_Allie@email.msn.com
|/ |LLIE | (313) 582-1540 |

Marc G. Fournier
Systems Administrator @ hub.org
primary: scrappy@hub.org secondary: scrappy@{freebsd|postgresql}.org

#26The Hermit Hacker
scrappy@hub.org
In reply to: Matthew N. Dodd (#21)
Re: [HACKERS] Open 6.4 items

On Thu, 29 Oct 1998, Matthew N. Dodd wrote:

On Thu, 29 Oct 1998, The Hermit Hacker wrote:

Solaris just doesn't have any mechanisms to work around the
limitation, I guess *shrug* It really sucks when you want to SIGHUP
the "parent process", which, under FreeBSD at least, is the one that
states: -accepting connections, but under Solaris they are *all* the
same :)

$ ps -eaf
UID PID PPID C STIME TTY TIME CMD
root 0 0 0 Oct 12 ? 0:01 sched
root 1 0 0 Oct 12 ? 0:15 /etc/init -
...

You'll note the 'PPID' field.

3 guesses what that stands for.

Okay, now you risk getting on my bad side :) I know what PPID
stands for...now you tell me which of these processes to SIGHUP:

root 18942 22213 0 13:22:03 ? 0:00 /usr/local/sbin/sendmail
root 18946 22213 0 13:22:03 ? 0:00 /usr/local/sbin/sendmail
root 18948 1 0 13:22:04 ? 0:00 /usr/local/sbin/sendmail
root 22213 1 0 Oct 26 ? 1:40 /usr/local/sbin/sendmail

And ya, I know, the one with the older date...the point is that
you can't really automate this, except to do:

kill -HUP `ps -aef | grep sendmail | awk '{print $2}'`

And SIGHUP them all...

#27The Hermit Hacker
scrappy@hub.org
In reply to: Noname (#22)
Re: [HACKERS] Open 6.4 items

On Thu, 29 Oct 1998, Jan Wieck wrote:

On Thu, 29 Oct 1998, The Hermit Hacker wrote:

Solaris just doesn't have any mechanisms to work around the
limitation, I guess *shrug* It really sucks when you want to SIGHUP
the "parent process", which, under FreeBSD at least, is the one that
states: -accepting connections, but under Solaris they are *all* the
same :)

$ ps -eaf
UID PID PPID C STIME TTY TIME CMD
root 0 0 0 Oct 12 ? 0:01 sched
root 1 0 0 Oct 12 ? 0:15 /etc/init -
...

You'll note the 'PPID' field.

3 guesses what that stands for.

Don't see how this is related to the topic - sorry.

PPID is the parent process ID. sched has no parent (it's a
kernel pseudo process) and init has sched as father. For all
other processes the PPID is set to init's PID at the time
their father dies (you'll see lot's of PPID=1).

But this all has nothing to do with changing the CMD column
of the ps output from inside a running process.

In Matthew's defence, I think the point he was trying to bring across was
that you should be able to look at hte PPID. sendmail, when you start,
tends to list its PPID as '1'...but, as I showed in my last email, that
doesn't appear to be "unique"...

Marc G. Fournier
Systems Administrator @ hub.org
primary: scrappy@hub.org secondary: scrappy@{freebsd|postgresql}.org

#28Bruce Momjian
maillist@candle.pha.pa.us
In reply to: Billy G. Allie (#18)
Re: [HACKERS] Open 6.4 items

Bruce Momjian wrote:

Doesn't work under Solaris, but, then again, neither does sendamil's, so
that isn't a shock :)

Man, if sendmail's doesn't work, that is really broken.

If you want strange, here is the output from the UnixWare 7 ps command with
some different options:

$ ps -ef | fgrep post
pgsql 10775 1 TS 80 0 01:34:12 ? 0:00 postmaster -S -i -o -F
pgsql 10869 10775 TS 80 0 01:45:30 ? 0:00 postmaster -S -i -o -F

$ ps -e | fgrep post
10775 TS 80 ? 0:00 postgres
10869 TS 80 ? 0:00 postgres

My guess is that ps without -f looks at the inode that opened the file,
and postmaster and postgres have the same inode because postmaster is
symlinked to postgres.

-- 
  Bruce Momjian                        |  http://www.op.net/~candle
  maillist@candle.pha.pa.us            |  (610) 853-3000
  +  If your life is a hard drive,     |  830 Blythe Avenue
  +  Christ can be your backup.        |  Drexel Hill, Pennsylvania 19026
#29Taral
taral@mail.utexas.edu
In reply to: The Hermit Hacker (#26)
RE: [HACKERS] Open 6.4 items

root 18942 22213 0 13:22:03 ? 0:00 /usr/local/sbin/sendmail
root 18946 22213 0 13:22:03 ? 0:00 /usr/local/sbin/sendmail
root 18948 1 0 13:22:04 ? 0:00 /usr/local/sbin/sendmail
root 22213 1 0 Oct 26 ? 1:40 /usr/local/sbin/sendmail

Er... if you have this, you have a problem. This means there are TWO
sendmail daemons running... There should only be ONE with a ppid of 1. It
looks like 18948 is blah.

Taral

#30Matthew N. Dodd
winter@jurai.net
In reply to: The Hermit Hacker (#26)
Re: [HACKERS] Open 6.4 items

On Thu, 29 Oct 1998, The Hermit Hacker wrote:

Okay, now you risk getting on my bad side :) I know what PPID
stands for...now you tell me which of these processes to SIGHUP:

root 18942 22213 0 13:22:03 ? 0:00 /usr/local/sbin/sendmail
root 18946 22213 0 13:22:03 ? 0:00 /usr/local/sbin/sendmail

2 open connections.

root 18948 1 0 13:22:04 ? 0:00 /usr/local/sbin/sendmail

a local delivery spawned by an open connection that closed and left the
local delivery to finish, thus orphaning it to init (PID1)

root 22213 1 0 Oct 26 ? 1:40 /usr/local/sbin/sendmail

The listener.

And ya, I know, the one with the older date...the point is that
you can't really automate this, except to do:

You're trying to kill the listener?

You do know you can use truss and lsof to figure this out as well right?

Or you could do:

kill -HUP `head -1 /var/run/sendmail.pid`

Also, check out what the BSD style ps shows under Solaris (/usr/ucb/ps).

--
| Matthew N. Dodd | 78 280Z | 75 164E | 84 245DL | FreeBSD/NetBSD/Sprite/VMS |
| winter@jurai.net | This Space For Rent | ix86,sparc,m68k,pmax,vax |
| http://www.jurai.net/~winter | Are you k-rad elite enough for my webpage? |

#31Matthew N. Dodd
winter@jurai.net
In reply to: Taral (#29)
RE: [HACKERS] Open 6.4 items

On Thu, 29 Oct 1998, Taral wrote:

root 18942 22213 0 13:22:03 ? 0:00 /usr/local/sbin/sendmail
root 18946 22213 0 13:22:03 ? 0:00 /usr/local/sbin/sendmail
root 18948 1 0 13:22:04 ? 0:00 /usr/local/sbin/sendmail
root 22213 1 0 Oct 26 ? 1:40 /usr/local/sbin/sendmail

Er... if you have this, you have a problem. This means there are TWO
sendmail daemons running... There should only be ONE with a ppid of 1. It
looks like 18948 is blah.

No, it looks like there are 4 sendmail daemons running.

The issue is to determine what each of them are doing.

daemon != listener

--
| Matthew N. Dodd | 78 280Z | 75 164E | 84 245DL | FreeBSD/NetBSD/Sprite/VMS |
| winter@jurai.net | This Space For Rent | ix86,sparc,m68k,pmax,vax |
| http://www.jurai.net/~winter | Are you k-rad elite enough for my webpage? |

#32Thomas G. Lockhart
lockhart@alumni.caltech.edu
In reply to: Matthew N. Dodd (#23)
Re: [HACKERS] Open 6.4 items

I really have to start explaining things using more words don't I.

Only if you want anyone to bother paying attention to you.

- Thomas (who didn't)

#33The Hermit Hacker
scrappy@hub.org
In reply to: Taral (#29)
RE: [HACKERS] Open 6.4 items

On Thu, 29 Oct 1998, Taral wrote:

root 18942 22213 0 13:22:03 ? 0:00 /usr/local/sbin/sendmail
root 18946 22213 0 13:22:03 ? 0:00 /usr/local/sbin/sendmail
root 18948 1 0 13:22:04 ? 0:00 /usr/local/sbin/sendmail
root 22213 1 0 Oct 26 ? 1:40 /usr/local/sbin/sendmail

Er... if you have this, you have a problem. This means there are TWO
sendmail daemons running... There should only be ONE with a ppid of 1. It
looks like 18948 is blah.

Good guess, but that is normal behaviour, at least under Solaris :) It
took me several 'ps -aef's to actually find it duplicated, it isn't always
there.

Marc G. Fournier
Systems Administrator @ hub.org
primary: scrappy@hub.org secondary: scrappy@{freebsd|postgresql}.org

#34The Hermit Hacker
scrappy@hub.org
In reply to: Matthew N. Dodd (#30)
Re: [HACKERS] Open 6.4 items

On Thu, 29 Oct 1998, Matthew N. Dodd wrote:

On Thu, 29 Oct 1998, The Hermit Hacker wrote:

Okay, now you risk getting on my bad side :) I know what PPID
stands for...now you tell me which of these processes to SIGHUP:

root 18942 22213 0 13:22:03 ? 0:00 /usr/local/sbin/sendmail
root 18946 22213 0 13:22:03 ? 0:00 /usr/local/sbin/sendmail

2 open connections.

root 18948 1 0 13:22:04 ? 0:00 /usr/local/sbin/sendmail

a local delivery spawned by an open connection that closed and left the
local delivery to finish, thus orphaning it to init (PID1)

root 22213 1 0 Oct 26 ? 1:40 /usr/local/sbin/sendmail

The listener.

And ya, I know, the one with the older date...the point is that
you can't really automate this, except to do:

You're trying to kill the listener?

You do know you can use truss and lsof to figure this out as well right?

Or you could do:

kill -HUP `head -1 /var/run/sendmail.pid`

Yes, of course, but this was not the point of the whole
thread...:)

Also, check out what the BSD style ps shows under Solaris (/usr/ucb/ps).

No apparent difference in the output...wait, you'll take me too
literally on that, eh? :) Both show the sasme information in the last
field../usr/local/sbin/sendmail -bd -q1h :)

Marc G. Fournier
Systems Administrator @ hub.org
primary: scrappy@hub.org secondary: scrappy@{freebsd|postgresql}.org

#35Hannu Krosing
hannu@trust.ee
In reply to: Noname (#22)
ORDER BY optimisations

Hallo Jan,

Do I remember right that your pathes to speed up ORDER BYs (by
omitting them when not needed) did not make it into 6.4 .

If that is the case, are they available anywhere ?

I really need them (fast) for my new project.

-------------
Hannu Krosing

#36Bruce Momjian
maillist@candle.pha.pa.us
In reply to: Hannu Krosing (#35)
Re: [HACKERS] ORDER BY optimisations

Hallo Jan,

Do I remember right that your pathes to speed up ORDER BYs (by
omitting them when not needed) did not make it into 6.4 .

If that is the case, are they available anywhere ?

I really need them (fast) for my new project.

LIMIT will probably be added to 6.4.1. Queries that use '%text%' can
not use indexes because they are not anchored at the beginning.
fulltextindex is in contrib for those cases.

-- 
  Bruce Momjian                        |  http://www.op.net/~candle
  maillist@candle.pha.pa.us            |  (610) 853-3000
  +  If your life is a hard drive,     |  830 Blythe Avenue
  +  Christ can be your backup.        |  Drexel Hill, Pennsylvania 19026
#37Hannu Krosing
hannu@trust.ee
In reply to: Bruce Momjian (#36)
Re: [HACKERS] ORDER BY optimisations

Bruce Momjian wrote:

Hallo Jan,

Do I remember right that your pathes to speed up ORDER BYs (by
omitting them when not needed) did not make it into 6.4 .

If that is the case, are they available anywhere ?

I really need them (fast) for my new project.

LIMIT will probably be added to 6.4.1.

Actually I don't need LIMIT that much (for me using CURSOR/MOVE/FETCH
is quite ok). The main benefit from LIMIT seems to be the ability
of giving the (future) optimiser a hint that we actulally need only
a small part of the whole query so it may be better to use an index.

What I am after, is the patch that removed the redundant sort node
when the access is already by an index matching the sort.

Queries that use '%text%' can not use indexes because they are not
anchored at the beginning.
fulltextindex is in contrib for those cases.

It still seems a bit of a cludge, although a useful one as its usage
is quite different from the use of other indexes. It also seems to be
quite liberal with wasting space as it makes both an additional table
_and_ an index for the words it indexes.

In fact I'm currently using my own fulltext indexing scheme outside
the database. I'm planning to work on including it in the pgsql
backend, once I figure out how the extending of access methods works.

--------------
Hannu

#38Noname
jwieck@debis.com
In reply to: Hannu Krosing (#35)
Re: [HACKERS] ORDER BY optimisations

Hallo Jan,

Do I remember right that your pathes to speed up ORDER BYs (by
omitting them when not needed) did not make it into 6.4 .

If that is the case, are they available anywhere ?

I really need them (fast) for my new project.

-------------
Hannu Krosing

Yepp, it didn't made it.

There where two different ones out, my and one from - hmmm -
was it Tatsuo or Hinoue? My one only suppresses the final
sort if

1. the plan is a Sort->IndexScan,

2. there is an ORDER BY clause,

3. the index choosen by the planner matches ALL attributes
given in the ORDER BY clause (extra indexed attributes
not in ORDER BY ignored),

4. and finally all sort operators are ASCENDING.

There are many debugging printf()'s in the patch and I think
one of them is still active while the others are commented
out. You need to comment out the last one yourself after you
found out that your queries are what causes it to suppress
the sorts.

Anyway, you said you need it fast, so here it is.

Jan

--

#======================================================================#
# It's easier to get forgiveness for being wrong than for being right. #
# Let's break this rule - forgive me. #
#======================================== jwieck@debis.com (Jan Wieck) #

diff -cr src.orig/backend/optimizer/plan/planner.c src/backend/optimizer/plan/planner.c
*** src.orig/backend/optimizer/plan/planner.c	Wed Oct 14 19:12:36 1998
--- src/backend/optimizer/plan/planner.c	Wed Oct 14 23:17:08 1998
***************
*** 48,53 ****
--- 48,59 ----

#include "executor/executor.h"

+ #include "utils/builtins.h"
+ #include "utils/syscache.h"
+ #include "access/genam.h"
+ #include "parser/parse_oper.h"
+ 
+ static bool need_sortplan(List *sortcls, Plan *plan);
  static Plan *make_sortplan(List *tlist, List *sortcls, Plan *plannode);
  extern Plan *make_groupPlan(List **tlist, bool tuplePerGroup,
  			   List *groupClause, Plan *subplan);
***************
*** 281,292 ****
  	}
  	else
  	{
! 		if (parse->sortClause)
  			return make_sortplan(tlist, parse->sortClause, result_plan);
  		else
  			return (Plan *) result_plan;
  	}

}

  /*
--- 287,450 ----
  	}
  	else
  	{
! 		if (parse->sortClause && need_sortplan(parse->sortClause, result_plan))
  			return make_sortplan(tlist, parse->sortClause, result_plan);
  		else
  			return (Plan *) result_plan;
  	}
+ }
+ 
+ static TargetEntry *
+ get_matching_tle(Plan *plan, Resdom *resdom)
+ {
+ 	List		*i;
+ 	TargetEntry	*tle;
+ 
+ 	foreach (i, plan->targetlist) {
+ 		tle = (TargetEntry *)lfirst(i);
+ 		if (tle->resdom->resno == resdom->resno)
+ 			return tle;
+ 	}
+ 	return NULL;
+ }
+ 
+ static bool
+ need_sortplan(List *sortcls, Plan *plan)
+ {
+ 	Relation	indexRel;
+ 	IndexScan	*indexScan;
+ 	Oid		indexId;
+ 	List		*i;
+ 	HeapTuple	htup;
+ 	Form_pg_index	index_tup;
+ 	int		key_no = 0;
+ 
+ 	/*
+ 	printf("check if need_sortplan ... ");
+ 	*/
+ 
+ 	if (nodeTag(plan) != T_IndexScan) {
+ 		/*
+ 		printf("not an index scan\n");
+ 		*/
+ 		return TRUE;
+ 	}
+ 
+ 	indexScan = (IndexScan *)plan;
+ 
+ 	if (plan->lefttree != NULL) {
+ 		/*
+ 		printf("scan has lefttree\n");
+ 		*/
+ 		return TRUE;
+ 	}
+ 	if (plan->righttree != NULL) {
+ 		/*
+ 		printf("scan has righttree\n");
+ 		*/
+ 		return TRUE;
+ 	}
+ 
+ 	if (length(indexScan->indxid) != 1) {
+ 		/*
+ 		printf("scanning multiple indices\n");
+ 		*/
+ 		return TRUE;
+ 	}
+ 
+ 	if (length(sortcls) > 8) {
+ 		/*
+ 		printf("sort clause too long (>8)\n");
+ 		*/
+ 		return TRUE;
+ 	}
+ 
+ 	indexId = lfirsti(indexScan->indxid);
+ 
+ 	indexRel = index_open(indexId);
+ 	if (strcmp(nameout(&(indexRel->rd_am->amname)), "btree") != 0) {
+ 		/*
+ 		printf("not a btree index\n");
+ 		*/
+ 		heap_close(indexRel);
+ 		return TRUE;
+ 	}
+ 	heap_close(indexRel);
+ 
+ 	htup = SearchSysCacheTuple(INDEXRELID,
+ 			ObjectIdGetDatum(indexId), 0, 0, 0);
+ 	if (!HeapTupleIsValid(htup)) {
+ 		elog(ERROR, "cache lookup for index %d failed", indexId);
+ 	}
+ 	index_tup = (Form_pg_index) GETSTRUCT(htup);
+ 
+ 	foreach (i, sortcls) {
+ 		SortClause	*sortcl;
+ 		Resdom		*resdom;
+ 		TargetEntry	*tle;
+ 		Var		*var;
+ 
+ 		sortcl = (SortClause *) lfirst(i);
+ 
+ 		/*
+ 		printf("\nchecking sortclause %s\n", nodeToString(sortcl));
+ 		*/
+ 
+ 		resdom = sortcl->resdom;
+ 		tle = get_matching_tle(plan, resdom);
+ 		if (tle == NULL) {
+ 			/*
+ 			printf("matching target entry not found\n");
+ 			*/
+ 			return TRUE;
+ 		}
+ 		if (nodeTag(tle->expr) != T_Var) {
+ 			/*
+ 			printf("target entry not a Var\n");
+ 			*/
+ 			return TRUE;
+ 		}
+ 		var = (Var *)(tle->expr);
+ 
+ 		if (var->varno != indexScan->scan.scanrelid) {
+ 			/*
+ 			printf("Var not from scanrelid\n");
+ 			*/
+ 			return TRUE;
+ 		}
+ 
+ 		if (var->varattno != index_tup->indkey[key_no]) {
+ 			/*
+ 			printf("attribute sorted does not match indexed att\n");
+ 			*/
+ 			return TRUE;
+ 		}
+ 
+ 		if (oprid(oper("<", resdom->restype, resdom->restype, FALSE)) != sortcl->opoid) {
+ 			/*
+ 			printf("opoid should be %d - is %d\n",
+ 				oprid(oper("<", resdom->restype, resdom->restype, FALSE)), sortcl->opoid);
+ 			*/
+ 			return TRUE;
+ 		}
+ 
+ 		key_no++;
+ 	}
+ 	if (key_no < 8 && index_tup->indkey[key_no] != 0) {
+ 		/*
+ 		printf("there are more indexed fields! ");
+ 		*/
+ 		return TRUE;
+ 	}
+ 
+ 	printf("SUPPRESSING sort over index scan\n");
+ 
+ 	/*
+ 	printf("scan = %s\n\n", nodeToString(indexScan));
+ 	*/
+ 
+ 	return FALSE;
  }

/*

#39Noname
jwieck@debis.com
In reply to: Noname (#38)
Re: [HACKERS] ORDER BY optimisations

1. the plan is a Sort->IndexScan,

2. there is an ORDER BY clause,

3. the index choosen by the planner matches ALL attributes
given in the ORDER BY clause (extra indexed attributes
not in ORDER BY ignored),

Ooops - sorry. Took another look at the patch and saw, that
it actually does not ignore extra attributes in the index.
Maybe you want to force sort suppression then too and comment
out the 'return TRUE' for this case.

BTW: This or an enhanced version (suppressing more stupid
sort cases) is my first candidate for the v6.4.1 feature
patch.

Jan

--

#======================================================================#
# It's easier to get forgiveness for being wrong than for being right. #
# Let's break this rule - forgive me. #
#======================================== jwieck@debis.com (Jan Wieck) #