diff --git a/contrib/pgcrypto/Makefile b/contrib/pgcrypto/Makefile index 18bad1a..04ce887 100644 --- a/contrib/pgcrypto/Makefile +++ b/contrib/pgcrypto/Makefile @@ -20,7 +20,7 @@ SRCS = pgcrypto.c px.c px-hmac.c px-crypt.c \ mbuf.c pgp.c pgp-armor.c pgp-cfb.c pgp-compress.c \ pgp-decrypt.c pgp-encrypt.c pgp-info.c pgp-mpi.c \ pgp-pubdec.c pgp-pubenc.c pgp-pubkey.c pgp-s2k.c \ - pgp-pgsql.c + pgp-pgsql.c xts.c MODULE_big = pgcrypto OBJS = $(SRCS:.c=.o) $(WIN32RES) diff --git a/contrib/pgcrypto/brg_endian.h b/contrib/pgcrypto/brg_endian.h new file mode 100644 index 0000000..e3cf0d1 --- /dev/null +++ b/contrib/pgcrypto/brg_endian.h @@ -0,0 +1,133 @@ +/* + --------------------------------------------------------------------------- + Copyright (c) 1998-2008, Brian Gladman, Worcester, UK. All rights reserved. + + LICENSE TERMS + + The redistribution and use of this software (with or without changes) + is allowed without the payment of fees or royalties provided that: + + 1. source code distributions include the above copyright notice, this + list of conditions and the following disclaimer; + + 2. binary distributions include the above copyright notice, this list + of conditions and the following disclaimer in their documentation; + + 3. the name of the copyright holder is not used to endorse products + built using this software without specific written permission. + + DISCLAIMER + + This software is provided 'as is' with no explicit or implied warranties + in respect of its properties, including, but not limited to, correctness + and/or fitness for purpose. + --------------------------------------------------------------------------- + Issue Date: 20/12/2007 +*/ + +#ifndef _BRG_ENDIAN_H +#define _BRG_ENDIAN_H + +#define IS_BIG_ENDIAN 4321 /* byte 0 is most significant (mc68k) */ +#define IS_LITTLE_ENDIAN 1234 /* byte 0 is least significant (i386) */ + +/* Include files where endian defines and byteswap functions may reside */ +#if defined( __sun ) +# include +#elif defined( __FreeBSD__ ) || defined( __OpenBSD__ ) || defined( __NetBSD__ ) +# include +#elif defined( BSD ) && ( BSD >= 199103 ) || defined( __APPLE__ ) || \ + defined( __CYGWIN32__ ) || defined( __DJGPP__ ) || defined( __osf__ ) +# include +#elif defined( __linux__ ) || defined( __GNUC__ ) || defined( __GNU_LIBRARY__ ) +# if !defined( __MINGW32__ ) && !defined( _AIX ) +# include +# if !defined( __BEOS__ ) +# include +# endif +# endif +#endif + +/* Now attempt to set the define for platform byte order using any */ +/* of the four forms SYMBOL, _SYMBOL, __SYMBOL & __SYMBOL__, which */ +/* seem to encompass most endian symbol definitions */ + +#if defined( BIG_ENDIAN ) && defined( LITTLE_ENDIAN ) +# if defined( BYTE_ORDER ) && BYTE_ORDER == BIG_ENDIAN +# define PLATFORM_BYTE_ORDER IS_BIG_ENDIAN +# elif defined( BYTE_ORDER ) && BYTE_ORDER == LITTLE_ENDIAN +# define PLATFORM_BYTE_ORDER IS_LITTLE_ENDIAN +# endif +#elif defined( BIG_ENDIAN ) +# define PLATFORM_BYTE_ORDER IS_BIG_ENDIAN +#elif defined( LITTLE_ENDIAN ) +# define PLATFORM_BYTE_ORDER IS_LITTLE_ENDIAN +#endif + +#if defined( _BIG_ENDIAN ) && defined( _LITTLE_ENDIAN ) +# if defined( _BYTE_ORDER ) && _BYTE_ORDER == _BIG_ENDIAN +# define PLATFORM_BYTE_ORDER IS_BIG_ENDIAN +# elif defined( _BYTE_ORDER ) && _BYTE_ORDER == _LITTLE_ENDIAN +# define PLATFORM_BYTE_ORDER IS_LITTLE_ENDIAN +# endif +#elif defined( _BIG_ENDIAN ) +# define PLATFORM_BYTE_ORDER IS_BIG_ENDIAN +#elif defined( _LITTLE_ENDIAN ) +# define PLATFORM_BYTE_ORDER IS_LITTLE_ENDIAN +#endif + +#if defined( __BIG_ENDIAN ) && defined( __LITTLE_ENDIAN ) +# if defined( __BYTE_ORDER ) && __BYTE_ORDER == __BIG_ENDIAN +# define PLATFORM_BYTE_ORDER IS_BIG_ENDIAN +# elif defined( __BYTE_ORDER ) && __BYTE_ORDER == __LITTLE_ENDIAN +# define PLATFORM_BYTE_ORDER IS_LITTLE_ENDIAN +# endif +#elif defined( __BIG_ENDIAN ) +# define PLATFORM_BYTE_ORDER IS_BIG_ENDIAN +#elif defined( __LITTLE_ENDIAN ) +# define PLATFORM_BYTE_ORDER IS_LITTLE_ENDIAN +#endif + +#if defined( __BIG_ENDIAN__ ) && defined( __LITTLE_ENDIAN__ ) +# if defined( __BYTE_ORDER__ ) && __BYTE_ORDER__ == __BIG_ENDIAN__ +# define PLATFORM_BYTE_ORDER IS_BIG_ENDIAN +# elif defined( __BYTE_ORDER__ ) && __BYTE_ORDER__ == __LITTLE_ENDIAN__ +# define PLATFORM_BYTE_ORDER IS_LITTLE_ENDIAN +# endif +#elif defined( __BIG_ENDIAN__ ) +# define PLATFORM_BYTE_ORDER IS_BIG_ENDIAN +#elif defined( __LITTLE_ENDIAN__ ) +# define PLATFORM_BYTE_ORDER IS_LITTLE_ENDIAN +#endif + +/* if the platform byte order could not be determined, then try to */ +/* set this define using common machine defines */ +#if !defined(PLATFORM_BYTE_ORDER) + +#if defined( __alpha__ ) || defined( __alpha ) || defined( i386 ) || \ + defined( __i386__ ) || defined( _M_I86 ) || defined( _M_IX86 ) || \ + defined( __OS2__ ) || defined( sun386 ) || defined( __TURBOC__ ) || \ + defined( vax ) || defined( vms ) || defined( VMS ) || \ + defined( __VMS ) || defined( _M_X64 ) +# define PLATFORM_BYTE_ORDER IS_LITTLE_ENDIAN + +#elif defined( AMIGA ) || defined( applec ) || defined( __AS400__ ) || \ + defined( _CRAY ) || defined( __hppa ) || defined( __hp9000 ) || \ + defined( ibm370 ) || defined( mc68000 ) || defined( m68k ) || \ + defined( __MRC__ ) || defined( __MVS__ ) || defined( __MWERKS__ ) || \ + defined( sparc ) || defined( __sparc) || defined( SYMANTEC_C ) || \ + defined( __VOS__ ) || defined( __TIGCC__ ) || defined( __TANDEM ) || \ + defined( THINK_C ) || defined( __VMCMS__ ) || defined( _AIX ) +# define PLATFORM_BYTE_ORDER IS_BIG_ENDIAN + +#elif 0 /* **** EDIT HERE IF NECESSARY **** */ +# define PLATFORM_BYTE_ORDER IS_LITTLE_ENDIAN +#elif 0 /* **** EDIT HERE IF NECESSARY **** */ +# define PLATFORM_BYTE_ORDER IS_BIG_ENDIAN +#else +# error Please edit lines 126 or 128 in brg_endian.h to set the platform byte order +#endif + +#endif + +#endif diff --git a/contrib/pgcrypto/brg_types.h b/contrib/pgcrypto/brg_types.h new file mode 100644 index 0000000..2675d2d --- /dev/null +++ b/contrib/pgcrypto/brg_types.h @@ -0,0 +1,223 @@ +/* + --------------------------------------------------------------------------- + Copyright (c) 1998-2008, Brian Gladman, Worcester, UK. All rights reserved. + + LICENSE TERMS + + The redistribution and use of this software (with or without changes) + is allowed without the payment of fees or royalties provided that: + + 1. source code distributions include the above copyright notice, this + list of conditions and the following disclaimer; + + 2. binary distributions include the above copyright notice, this list + of conditions and the following disclaimer in their documentation; + + 3. the name of the copyright holder is not used to endorse products + built using this software without specific written permission. + + DISCLAIMER + + This software is provided 'as is' with no explicit or implied warranties + in respect of its properties, including, but not limited to, correctness + and/or fitness for purpose. + --------------------------------------------------------------------------- + Issue Date: 20/12/2007 + + The unsigned integer types defined here are of the form uint_t where + is the length of the type; for example, the unsigned 32-bit type is + 'uint_32t'. These are NOT the same as the 'C99 integer types' that are + defined in the inttypes.h and stdint.h headers since attempts to use these + types have shown that support for them is still highly variable. However, + since the latter are of the form uint_t, a regular expression search + and replace (in VC++ search on 'uint_{:z}t' and replace with 'uint\1_t') + can be used to convert the types used here to the C99 standard types. +*/ + +#ifndef _BRG_TYPES_H +#define _BRG_TYPES_H + +#if defined(__cplusplus) +extern "C" { +#endif + +#include + +#if defined( _MSC_VER ) && ( _MSC_VER >= 1300 ) +# include +# define ptrint_t intptr_t +#elif defined( __GNUC__ ) && ( __GNUC__ >= 3 ) +# include +# define ptrint_t intptr_t +#else +# define ptrint_t int +#endif + +#ifndef BRG_UI8 +# define BRG_UI8 +# if UCHAR_MAX == 255u + typedef unsigned char uint_8t; +# else +# error Please define uint_8t as an 8-bit unsigned integer type in brg_types.h +# endif +#endif + +#ifndef BRG_UI16 +# define BRG_UI16 +# if USHRT_MAX == 65535u + typedef unsigned short uint_16t; +# else +# error Please define uint_16t as a 16-bit unsigned short type in brg_types.h +# endif +#endif + +#ifndef BRG_UI32 +# define BRG_UI32 +# if UINT_MAX == 4294967295u +# define li_32(h) 0x##h##u + typedef unsigned int uint_32t; +# elif ULONG_MAX == 4294967295u +# define li_32(h) 0x##h##ul + typedef unsigned long uint_32t; +# elif defined( _CRAY ) +# error This code needs 32-bit data types, which Cray machines do not provide +# else +# error Please define uint_32t as a 32-bit unsigned integer type in brg_types.h +# endif +#endif + +#ifndef BRG_UI64 +# if defined( __BORLANDC__ ) && !defined( __MSDOS__ ) +# define BRG_UI64 +# define li_64(h) 0x##h##ui64 + typedef unsigned __int64 uint_64t; +# elif defined( _MSC_VER ) && ( _MSC_VER < 1300 ) /* 1300 == VC++ 7.0 */ +# define BRG_UI64 +# define li_64(h) 0x##h##ui64 + typedef unsigned __int64 uint_64t; +# elif defined( __sun ) && defined(ULONG_MAX) && ULONG_MAX == 0xfffffffful +# define BRG_UI64 +# define li_64(h) 0x##h##ull + typedef unsigned long long uint_64t; +# elif defined( __MVS__ ) +# define BRG_UI64 +# define li_64(h) 0x##h##ull + typedef unsigned int long long uint_64t; +# elif defined( UINT_MAX ) && UINT_MAX > 4294967295u +# if UINT_MAX == 18446744073709551615u +# define BRG_UI64 +# define li_64(h) 0x##h##u + typedef unsigned int uint_64t; +# endif +# elif defined( ULONG_MAX ) && ULONG_MAX > 4294967295u +# if ULONG_MAX == 18446744073709551615ul +# define BRG_UI64 +# define li_64(h) 0x##h##ul + typedef unsigned long uint_64t; +# endif +# elif defined( ULLONG_MAX ) && ULLONG_MAX > 4294967295u +# if ULLONG_MAX == 18446744073709551615ull +# define BRG_UI64 +# define li_64(h) 0x##h##ull + typedef unsigned long long uint_64t; +# endif +# elif defined( ULONG_LONG_MAX ) && ULONG_LONG_MAX > 4294967295u +# if ULONG_LONG_MAX == 18446744073709551615ull +# define BRG_UI64 +# define li_64(h) 0x##h##ull + typedef unsigned long long uint_64t; +# endif +# endif +#endif + +#if !defined( BRG_UI64 ) +# if defined( NEED_UINT_64T ) +# error Please define uint_64t as an unsigned 64 bit type in brg_types.h +# endif +#endif + +#ifndef RETURN_VALUES +# define RETURN_VALUES +# if defined( DLL_EXPORT ) +# if defined( _MSC_VER ) || defined ( __INTEL_COMPILER ) +# define VOID_RETURN __declspec( dllexport ) void __stdcall +# define INT_RETURN __declspec( dllexport ) int __stdcall +# elif defined( __GNUC__ ) +# define VOID_RETURN __declspec( __dllexport__ ) void +# define INT_RETURN __declspec( __dllexport__ ) int +# else +# error Use of the DLL is only available on the Microsoft, Intel and GCC compilers +# endif +# elif defined( DLL_IMPORT ) +# if defined( _MSC_VER ) || defined ( __INTEL_COMPILER ) +# define VOID_RETURN __declspec( dllimport ) void __stdcall +# define INT_RETURN __declspec( dllimport ) int __stdcall +# elif defined( __GNUC__ ) +# define VOID_RETURN __declspec( __dllimport__ ) void +# define INT_RETURN __declspec( __dllimport__ ) int +# else +# error Use of the DLL is only available on the Microsoft, Intel and GCC compilers +# endif +# elif defined( __WATCOMC__ ) +# define VOID_RETURN void __cdecl +# define INT_RETURN int __cdecl +# else +# define VOID_RETURN void +# define INT_RETURN int +# endif +#endif + +/* These defines are used to detect and set the memory alignment of pointers. + Note that offsets are in bytes. + + ALIGN_OFFSET(x,n) return the positive or zero offset of + the memory addressed by the pointer 'x' + from an address that is aligned on an + 'n' byte boundary ('n' is a power of 2) + + ALIGN_FLOOR(x,n) return a pointer that points to memory + that is aligned on an 'n' byte boundary + and is not higher than the memory address + pointed to by 'x' ('n' is a power of 2) + + ALIGN_CEIL(x,n) return a pointer that points to memory + that is aligned on an 'n' byte boundary + and is not lower than the memory address + pointed to by 'x' ('n' is a power of 2) +*/ + +#define ALIGN_OFFSET(x,n) (((ptrint_t)(x)) & ((n) - 1)) +#define ALIGN_FLOOR(x,n) ((uint_8t*)(x) - ( ((ptrint_t)(x)) & ((n) - 1))) +#define ALIGN_CEIL(x,n) ((uint_8t*)(x) + (-((ptrint_t)(x)) & ((n) - 1))) + +/* These defines are used to declare buffers in a way that allows + faster operations on longer variables to be used. In all these + defines 'size' must be a power of 2 and >= 8. NOTE that the + buffer size is in bytes but the type length is in bits + + UNIT_TYPEDEF(x,size) declares a variable 'x' of length + 'size' bits + + BUFR_TYPEDEF(x,size,bsize) declares a buffer 'x' of length 'bsize' + bytes defined as an array of variables + each of 'size' bits (bsize must be a + multiple of size / 8) + + UNIT_CAST(x,size) casts a variable to a type of + length 'size' bits + + UPTR_CAST(x,size) casts a pointer to a pointer to a + varaiable of length 'size' bits +*/ + +#define UI_TYPE(size) uint_##size##t +#define UNIT_TYPEDEF(x,size) typedef UI_TYPE(size) x +#define BUFR_TYPEDEF(x,size,bsize) typedef UI_TYPE(size) x[bsize / (size >> 3)] +#define UNIT_CAST(x,size) ((UI_TYPE(size) )(x)) +#define UPTR_CAST(x,size) ((UI_TYPE(size)*)(x)) + +#if defined(__cplusplus) +} +#endif + +#endif diff --git a/contrib/pgcrypto/mode_hdr.h b/contrib/pgcrypto/mode_hdr.h new file mode 100644 index 0000000..46bd94f --- /dev/null +++ b/contrib/pgcrypto/mode_hdr.h @@ -0,0 +1,323 @@ +/* + --------------------------------------------------------------------------- + Copyright (c) 1998-2008, Brian Gladman, Worcester, UK. All rights reserved. + + LICENSE TERMS + + The redistribution and use of this software (with or without changes) + is allowed without the payment of fees or royalties provided that: + + 1. source code distributions include the above copyright notice, this + list of conditions and the following disclaimer; + + 2. binary distributions include the above copyright notice, this list + of conditions and the following disclaimer in their documentation; + + 3. the name of the copyright holder is not used to endorse products + built using this software without specific written permission. + + DISCLAIMER + + This software is provided 'as is' with no explicit or implied warranties + in respect of its properties, including, but not limited to, correctness + and/or fitness for purpose. + --------------------------------------------------------------------------- + Issue Date: 20/12/2007 + + This header file is an INTERNAL file which supports mode implementation +*/ + +#ifndef _MODE_HDR_H +#define _MODE_HDR_H + +#include +#include + +#include "brg_endian.h" + +/* This define sets the units in which buffers are processed. This code + can provide significant speed gains if buffers can be processed in + 32 or 64 bit chunks rather than in bytes. This define sets the units + in which buffers will be accessed if possible +*/ +#if !defined( UNIT_BITS ) +# if 1 +# define UNIT_BITS 64 +# elif 0 +# define UNIT_BITS 32 +# else +# define UNIT_BITS 8 +# endif +#endif + +#if UNIT_BITS == 64 && !defined( NEED_UINT_64T ) +# define NEED_UINT_64T +#endif + +#include "brg_types.h" + +/* Use of inlines is preferred but code blocks can also be expanded inline + using 'defines'. But the latter approach will typically generate a LOT + of code and is not recommended. +*/ +#if 1 && !defined( USE_INLINING ) +# define USE_INLINING +#endif + +#if defined( _MSC_VER ) +# if _MSC_VER >= 1400 +# include +# include +# pragma intrinsic(memset) +# pragma intrinsic(memcpy) +# define rotl32 _rotl +# define rotr32 _rotr +# define rotl64 _rotl64 +# define rotr64 _rotl64 +# define bswap_16(x) _byteswap_ushort(x) +# define bswap_32(x) _byteswap_ulong(x) +# define bswap_64(x) _byteswap_uint64(x) +# else +# define rotl32 _lrotl +# define rotr32 _lrotr +# endif +#endif + +#if defined( USE_INLINING ) +# if defined( _MSC_VER ) +# define mh_decl __inline +# elif defined( __GNUC__ ) || defined( __GNU_LIBRARY__ ) +# define mh_decl static inline +# else +# define mh_decl static +# endif +#endif + +#if defined(__cplusplus) +extern "C" { +#endif + +#define UI8_PTR(x) UPTR_CAST(x, 8) +#define UI16_PTR(x) UPTR_CAST(x, 16) +#define UI32_PTR(x) UPTR_CAST(x, 32) +#define UI64_PTR(x) UPTR_CAST(x, 64) +#define UNIT_PTR(x) UPTR_CAST(x, UNIT_BITS) + +#define BUF_INC (UNIT_BITS >> 3) +#define BUF_ADRMASK ((UNIT_BITS >> 3) - 1) + +#define rep2_u2(f,r,x) f( 0,r,x); f( 1,r,x) +#define rep2_u4(f,r,x) f( 0,r,x); f( 1,r,x); f( 2,r,x); f( 3,r,x) +#define rep2_u16(f,r,x) f( 0,r,x); f( 1,r,x); f( 2,r,x); f( 3,r,x); \ + f( 4,r,x); f( 5,r,x); f( 6,r,x); f( 7,r,x); \ + f( 8,r,x); f( 9,r,x); f(10,r,x); f(11,r,x); \ + f(12,r,x); f(13,r,x); f(14,r,x); f(15,r,x) + +#define rep2_d2(f,r,x) f( 1,r,x); f( 0,r,x) +#define rep2_d4(f,r,x) f( 3,r,x); f( 2,r,x); f( 1,r,x); f( 0,r,x) +#define rep2_d16(f,r,x) f(15,r,x); f(14,r,x); f(13,r,x); f(12,r,x); \ + f(11,r,x); f(10,r,x); f( 9,r,x); f( 8,r,x); \ + f( 7,r,x); f( 6,r,x); f( 5,r,x); f( 4,r,x); \ + f( 3,r,x); f( 2,r,x); f( 1,r,x); f( 0,r,x) + +#define rep3_u2(f,r,x,y) f( 0,r,x,y); f( 1,r,x,y) +#define rep3_u4(f,r,x,y) f( 0,r,x,y); f( 1,r,x,y); f( 2,r,x,y); f( 3,r,x,y) +#define rep3_u16(f,r,x,y) f( 0,r,x,y); f( 1,r,x,y); f( 2,r,x,y); f( 3,r,x,y); \ + f( 4,r,x,y); f( 5,r,x,y); f( 6,r,x,y); f( 7,r,x,y); \ + f( 8,r,x,y); f( 9,r,x,y); f(10,r,x,y); f(11,r,x,y); \ + f(12,r,x,y); f(13,r,x,y); f(14,r,x,y); f(15,r,x,y) + +#define rep3_d2(f,r,x,y) f( 1,r,x,y); f( 0,r,x,y) +#define rep3_d4(f,r,x,y) f( 3,r,x,y); f( 2,r,x,y); f( 1,r,x,y); f( 0,r,x,y) +#define rep3_d16(f,r,x,y) f(15,r,x,y); f(14,r,x,y); f(13,r,x,y); f(12,r,x,y); \ + f(11,r,x,y); f(10,r,x,y); f( 9,r,x,y); f( 8,r,x,y); \ + f( 7,r,x,y); f( 6,r,x,y); f( 5,r,x,y); f( 4,r,x,y); \ + f( 3,r,x,y); f( 2,r,x,y); f( 1,r,x,y); f( 0,r,x,y) + +/* function pointers might be used for fast XOR operations */ + +typedef void (*xor_function)(void* r, const void* p, const void* q); + +/* left and right rotates on 32 and 64 bit variables */ + +#if !defined( rotl32 ) // NOTE: 0 <= n <= 32 ASSUMED +mh_decl uint_32t rotl32(uint_32t x, int n) +{ + return (((x) << n) | ((x) >> (32 - n))); +} +#endif + +#if !defined( rotr32 ) // NOTE: 0 <= n <= 32 ASSUMED +mh_decl uint_32t rotr32(uint_32t x, int n) +{ + return (((x) >> n) | ((x) << (32 - n))); +} +#endif + +#if !defined( rotl64 ) // NOTE: 0 <= n <= 64 ASSUMED +mh_decl uint_64t rotl64(uint_64t x, int n) +{ + return (((x) << n) | ((x) >> (64 - n))); +} +#endif + +#if !defined( rotr64 ) // NOTE: 0 <= n <= 64 ASSUMED +mh_decl uint_64t rotr64(uint_64t x, int n) +{ + return (((x) >> n) | ((x) << (64 - n))); +} +#endif + +/* byte order inversions for 16, 32 and 64 bit variables */ + +#if !defined(bswap_16) +mh_decl uint_16t bswap_16(uint_16t x) +{ + return (x >> 8) | (x << 8); +} +#endif + +#if !defined(bswap_32) +mh_decl uint_32t bswap_32(uint_32t x) +{ + return ((rotr32((x), 24) & 0x00ff00ff) | (rotr32((x), 8) & 0xff00ff00)); +} +#endif + +#if !defined(bswap_64) +mh_decl uint_64t bswap_64(uint_64t x) +{ + return bswap_32((uint_32t)(x >> 32)) | ((uint_64t)bswap_32((uint_32t)x) << 32); +} +#endif +/* support for fast aligned buffer move, xor and byte swap operations - + source and destination buffers for move and xor operations must not + overlap, those for byte order revesal must either not overlap or + must be identical +*/ +#define f_copy(n,p,q) p[n] = q[n] +#define f_xor(n,r,p,q) r[n] = p[n] ^ q[n] + +mh_decl void copy_block(void* p, const void* q) +{ + memcpy(p, q, 16); +} + +mh_decl void copy_block_aligned(void *p, const void *q) +{ +#if UNIT_BITS == 8 + memcpy(p, q, 16); +#elif UNIT_BITS == 32 + rep2_u4(f_copy,UNIT_PTR(p),UNIT_PTR(q)); +#else + rep2_u2(f_copy,UNIT_PTR(p),UNIT_PTR(q)); +#endif +} + +mh_decl void xor_block(void *r, const void* p, const void* q) +{ + rep3_u16(f_xor, UI8_PTR(r), UI8_PTR(p), UI8_PTR(q)); +} + +mh_decl void xor_block_aligned(void *r, const void *p, const void *q) +{ +#if UNIT_BITS == 8 + rep3_u16(f_xor, UNIT_PTR(r), UNIT_PTR(p), UNIT_PTR(q)); +#elif UNIT_BITS == 32 + rep3_u4(f_xor, UNIT_PTR(r), UNIT_PTR(p), UNIT_PTR(q)); +#else + rep3_u2(f_xor, UNIT_PTR(r), UNIT_PTR(p), UNIT_PTR(q)); +#endif +} + +mh_decl void bswap32_block(void *d, const void* s) +{ +#if UNIT_BITS == 8 + uint_8t t; + t = UNIT_PTR(s)[ 0]; UNIT_PTR(d)[ 0] = UNIT_PTR(s)[ 3]; UNIT_PTR(d)[ 3] = t; + t = UNIT_PTR(s)[ 1]; UNIT_PTR(d)[ 1] = UNIT_PTR(s)[ 2]; UNIT_PTR(d)[ 2] = t; + t = UNIT_PTR(s)[ 4]; UNIT_PTR(d)[ 4] = UNIT_PTR(s)[ 7]; UNIT_PTR(d)[ 7] = t; + t = UNIT_PTR(s)[ 5]; UNIT_PTR(d)[ 5] = UNIT_PTR(s)[ 6]; UNIT_PTR(d) [6] = t; + t = UNIT_PTR(s)[ 8]; UNIT_PTR(d)[ 8] = UNIT_PTR(s)[11]; UNIT_PTR(d)[12] = t; + t = UNIT_PTR(s)[ 9]; UNIT_PTR(d)[ 9] = UNIT_PTR(s)[10]; UNIT_PTR(d)[10] = t; + t = UNIT_PTR(s)[12]; UNIT_PTR(d)[12] = UNIT_PTR(s)[15]; UNIT_PTR(d)[15] = t; + t = UNIT_PTR(s)[13]; UNIT_PTR(d)[ 3] = UNIT_PTR(s)[14]; UNIT_PTR(d)[14] = t; +#elif UNIT_BITS == 32 + UNIT_PTR(d)[0] = bswap_32(UNIT_PTR(s)[0]); UNIT_PTR(d)[1] = bswap_32(UNIT_PTR(s)[1]); + UNIT_PTR(d)[2] = bswap_32(UNIT_PTR(s)[2]); UNIT_PTR(d)[3] = bswap_32(UNIT_PTR(s)[3]); +#else + UI32_PTR(d)[0] = bswap_32(UI32_PTR(s)[0]); UI32_PTR(d)[1] = bswap_32(UI32_PTR(s)[1]); + UI32_PTR(d)[2] = bswap_32(UI32_PTR(s)[2]); UI32_PTR(d)[3] = bswap_32(UI32_PTR(s)[3]); +#endif +} + +mh_decl void bswap64_block(void *d, const void* s) +{ +#if UNIT_BITS == 8 + uint_8t t; + t = UNIT_PTR(s)[ 0]; UNIT_PTR(d)[ 0] = UNIT_PTR(s)[ 7]; UNIT_PTR(d)[ 7] = t; + t = UNIT_PTR(s)[ 1]; UNIT_PTR(d)[ 1] = UNIT_PTR(s)[ 6]; UNIT_PTR(d)[ 6] = t; + t = UNIT_PTR(s)[ 2]; UNIT_PTR(d)[ 2] = UNIT_PTR(s)[ 5]; UNIT_PTR(d)[ 5] = t; + t = UNIT_PTR(s)[ 3]; UNIT_PTR(d)[ 3] = UNIT_PTR(s)[ 3]; UNIT_PTR(d) [3] = t; + t = UNIT_PTR(s)[ 8]; UNIT_PTR(d)[ 8] = UNIT_PTR(s)[15]; UNIT_PTR(d)[15] = t; + t = UNIT_PTR(s)[ 9]; UNIT_PTR(d)[ 9] = UNIT_PTR(s)[14]; UNIT_PTR(d)[14] = t; + t = UNIT_PTR(s)[10]; UNIT_PTR(d)[10] = UNIT_PTR(s)[13]; UNIT_PTR(d)[13] = t; + t = UNIT_PTR(s)[11]; UNIT_PTR(d)[11] = UNIT_PTR(s)[12]; UNIT_PTR(d)[12] = t; +#elif UNIT_BITS == 32 + uint_32t t; + t = bswap_32(UNIT_PTR(s)[0]); UNIT_PTR(d)[0] = bswap_32(UNIT_PTR(s)[1]); UNIT_PTR(d)[1] = t; + t = bswap_32(UNIT_PTR(s)[2]); UNIT_PTR(d)[2] = bswap_32(UNIT_PTR(s)[2]); UNIT_PTR(d)[3] = t; +#else + UNIT_PTR(d)[0] = bswap_64(UNIT_PTR(s)[0]); UNIT_PTR(d)[1] = bswap_64(UNIT_PTR(s)[1]); +#endif +} + +mh_decl void bswap128_block(void *d, const void* s) +{ +#if UNIT_BITS == 8 + uint_8t t; + t = UNIT_PTR(s)[0]; UNIT_PTR(d)[0] = UNIT_PTR(s)[15]; UNIT_PTR(d)[15] = t; + t = UNIT_PTR(s)[1]; UNIT_PTR(d)[1] = UNIT_PTR(s)[14]; UNIT_PTR(d)[14] = t; + t = UNIT_PTR(s)[2]; UNIT_PTR(d)[2] = UNIT_PTR(s)[13]; UNIT_PTR(d)[13] = t; + t = UNIT_PTR(s)[3]; UNIT_PTR(d)[3] = UNIT_PTR(s)[12]; UNIT_PTR(d)[12] = t; + t = UNIT_PTR(s)[4]; UNIT_PTR(d)[4] = UNIT_PTR(s)[11]; UNIT_PTR(d)[11] = t; + t = UNIT_PTR(s)[5]; UNIT_PTR(d)[5] = UNIT_PTR(s)[10]; UNIT_PTR(d)[10] = t; + t = UNIT_PTR(s)[6]; UNIT_PTR(d)[6] = UNIT_PTR(s)[ 9]; UNIT_PTR(d)[ 9] = t; + t = UNIT_PTR(s)[7]; UNIT_PTR(d)[7] = UNIT_PTR(s)[ 8]; UNIT_PTR(d)[ 8] = t; +#elif UNIT_BITS == 32 + uint_32t t; + t = bswap_32(UNIT_PTR(s)[0]); UNIT_PTR(d)[0] = bswap_32(UNIT_PTR(s)[3]); UNIT_PTR(d)[3] = t; + t = bswap_32(UNIT_PTR(s)[1]); UNIT_PTR(d)[1] = bswap_32(UNIT_PTR(s)[2]); UNIT_PTR(d)[2] = t; +#else + uint_64t t; + t = bswap_64(UNIT_PTR(s)[0]); UNIT_PTR(d)[0] = bswap_64(UNIT_PTR(s)[1]); UNIT_PTR(d)[1] = t; +#endif +} + +/* platform byte order to big or little endian order for 16, 32 and 64 bit variables */ + +#if PLATFORM_BYTE_ORDER == IS_BIG_ENDIAN + +# define uint_16t_to_le(x) (x) = bswap_16((x)) +# define uint_32t_to_le(x) (x) = bswap_32((x)) +# define uint_64t_to_le(x) (x) = bswap_64((x)) +# define uint_16t_to_be(x) +# define uint_32t_to_be(x) +# define uint_64t_to_be(x) + +#else + +# define uint_16t_to_le(x) +# define uint_32t_to_le(x) +# define uint_64t_to_le(x) +# define uint_16t_to_be(x) (x) = bswap_16((x)) +# define uint_32t_to_be(x) (x) = bswap_32((x)) +# define uint_64t_to_be(x) (x) = bswap_64((x)) + +#endif + +#if defined(__cplusplus) +} +#endif + +#endif diff --git a/contrib/pgcrypto/pgcrypto.c b/contrib/pgcrypto/pgcrypto.c index 2d446d8..8599203 100644 --- a/contrib/pgcrypto/pgcrypto.c +++ b/contrib/pgcrypto/pgcrypto.c @@ -34,12 +34,15 @@ #include #include "parser/scansup.h" +#include "storage/encryption.h" #include "utils/builtins.h" #include "utils/uuid.h" #include "px.h" #include "px-crypt.h" #include "pgcrypto.h" +#include "sha2.h" +#include "xts.h" PG_MODULE_MAGIC; @@ -47,6 +50,23 @@ PG_MODULE_MAGIC; typedef int (*PFN) (const char *name, void **res); static void *find_provider(text *name, PFN pf, char *desc, int silent); +static bool pgcrypto_encryption_setup(); +static void pgcrypto_encrypt_block(const char *input, + char *output, Size size, const char *tweak); +static void pgcrypto_decrypt_block(const char *input, + char *output, Size size, const char *tweak); +void _PG_init(void); + +/* + * Encryption and decryption keys for full database encryption support. + */ +typedef struct { + xts_encrypt_ctx enc_ctx[1]; + xts_decrypt_ctx dec_ctx[1]; +} db_encryption_ctx; + +/* Full database encryption key, initialized by pgcrypto_encryption_setup. */ +static db_encryption_ctx db_key; /* SQL function: hash(bytea, text) returns bytea */ PG_FUNCTION_INFO_V1(pg_digest); @@ -494,3 +514,72 @@ find_provider(text *name, return err ? NULL : res; } + +/* + * Pgcrypto module does AES-128-XTS encryption. + */ +static bool +pgcrypto_encryption_setup() +{ + uint8 key[32]; + char *passphrase = getenv("PGENCRYPTIONKEY"); + + /* Empty or missing passphrase means that encryption is not configured */ + if (passphrase == NULL || passphrase[0] == '\0') + { + ereport(LOG, + (errmsg("encryption key not provided"), + errdetail("The database cluster was initialized with encryption" + " but the server was started without an encryption key."), + errhint("Set the key using PGENCRYPTIONKEY environment variable."))); + return false; + } + + /* TODO: replace with PBKDF2 or scrypt */ + { + SHA256_CTX sha_ctx; + SHA256_Init(&sha_ctx); + SHA256_Update(&sha_ctx, (uint8*) passphrase, strlen(passphrase)); + SHA256_Final(key, &sha_ctx); + } + + if (xts_encrypt_key(key, 32, db_key.enc_ctx) != EXIT_SUCCESS || + xts_decrypt_key(key, 32, db_key.dec_ctx) != EXIT_SUCCESS) + { + elog(ERROR, "Encryption key setup failed."); + return false; + } + + return true; +} + +static void +pgcrypto_encrypt_block(const char *input, char *output, Size size, + const char *tweak) +{ + if (input != output) + memcpy(output, input, size); + + xts_encrypt_block((uint8*) output, (const uint8*) tweak, size, db_key.enc_ctx); +} + +static void +pgcrypto_decrypt_block(const char *input, char *output, Size size, + const char *tweak) +{ + if (input != output) + memcpy(output, input, size); + + xts_decrypt_block((uint8*) output, (const uint8*) tweak, size, db_key.dec_ctx); +} + +void +_PG_init(void) +{ + EncryptionRoutines routines; + routines.SetupEncryption = &pgcrypto_encryption_setup; + routines.EncryptBlock = &pgcrypto_encrypt_block; + routines.DecryptBlock = &pgcrypto_decrypt_block; + + register_encryption_module("pgcrypto", &routines); +} diff --git a/contrib/pgcrypto/xts.c b/contrib/pgcrypto/xts.c new file mode 100644 index 0000000..4efc2bc --- /dev/null +++ b/contrib/pgcrypto/xts.c @@ -0,0 +1,220 @@ +/* + --------------------------------------------------------------------------- + Copyright (c) 1998-2008, Brian Gladman, Worcester, UK. All rights reserved. + + LICENSE TERMS + + The redistribution and use of this software (with or without changes) + is allowed without the payment of fees or royalties provided that: + + 1. source code distributions include the above copyright notice, this + list of conditions and the following disclaimer; + + 2. binary distributions include the above copyright notice, this list + of conditions and the following disclaimer in their documentation; + + 3. the name of the copyright holder is not used to endorse products + built using this software without specific written permission. + + DISCLAIMER + + This software is provided 'as is' with no explicit or implied warranties + in respect of its properties, including, but not limited to, correctness + and/or fitness for purpose. + --------------------------------------------------------------------------- + Issue Date: 20/12/2007 + + My thanks to both Doug Whiting and Olaf Pors for their much appreciated + assistance in debugging and testing this code. +*/ + +#include "postgres.h" + +#include "mode_hdr.h" +#include "xts.h" + +static void gf_mulx(void *x); + +UNIT_TYPEDEF(buf_unit, UNIT_BITS); +BUFR_TYPEDEF(buf_type, UNIT_BITS, AES_BLOCK_SIZE); + +static void gf_mulx(void *x) +{ +#if UNIT_BITS == 8 + + uint_8t i = 16, t = ((uint_8t*)x)[15]; + while(--i) + ((uint_8t*)x)[i] = (((uint_8t*)x)[i] << 1) | (((uint_8t*)x)[i - 1] & 0x80 ? 1 : 0); + ((uint_8t*)x)[0] = (((uint_8t*)x)[0] << 1) ^ (t & 0x80 ? 0x87 : 0x00); + +#elif PLATFORM_BYTE_ORDER == IS_LITTLE_ENDIAN + +# if UNIT_BITS == 64 + +# define GF_MASK li_64(8000000000000000) +# define GF_XOR li_64(0000000000000087) + uint_64t _tt = ((UPTR_CAST(x,64)[1] & GF_MASK) ? GF_XOR : 0); + UPTR_CAST(x,64)[1] = (UPTR_CAST(x,64)[1] << 1) | (UPTR_CAST(x,64)[0] & GF_MASK ? 1 : 0); + UPTR_CAST(x,64)[0] = (UPTR_CAST(x,64)[0] << 1) ^ _tt; + +# else /* UNIT_BITS == 32 */ + +# define GF_MASK li_32(80000000) +# define GF_XOR li_32(00000087) + uint_32t _tt = ((UPTR_CAST(x,32)[3] & GF_MASK) ? GF_XOR : 0);; + UPTR_CAST(x,32)[3] = (UPTR_CAST(x,32)[3] << 1) | (UPTR_CAST(x,32)[2] & GF_MASK ? 1 : 0); + UPTR_CAST(x,32)[2] = (UPTR_CAST(x,32)[2] << 1) | (UPTR_CAST(x,32)[1] & GF_MASK ? 1 : 0); + UPTR_CAST(x,32)[1] = (UPTR_CAST(x,32)[1] << 1) | (UPTR_CAST(x,32)[0] & GF_MASK ? 1 : 0); + UPTR_CAST(x,32)[0] = (UPTR_CAST(x,32)[0] << 1) ^ _tt; + +# endif + +#else /* PLATFORM_BYTE_ORDER == IS_BIG_ENDIAN */ + +# if UNIT_BITS == 64 + +# define MASK_01 li_64(0101010101010101) +# define GF_MASK li_64(0000000000000080) +# define GF_XOR li_64(8700000000000000) + uint_64t _tt = ((UPTR_CAST(x,64)[1] & GF_MASK) ? GF_XOR : 0); + UPTR_CAST(x,64)[1] = ((UPTR_CAST(x,64)[1] << 1) & ~MASK_01) + | (((UPTR_CAST(x,64)[1] >> 15) | (UPTR_CAST(x,64)[0] << 49)) & MASK_01); + UPTR_CAST(x,64)[0] = (((UPTR_CAST(x,64)[0] << 1) & ~MASK_01) + | ((UPTR_CAST(x,64)[0] >> 15) & MASK_01)) ^ _tt; + +# else /* UNIT_BITS == 32 */ + +# define MASK_01 li_32(01010101) +# define GF_MASK li_32(00000080) +# define GF_XOR li_32(87000000) + uint_32t _tt = ((UPTR_CAST(x,32)[3] & GF_MASK) ? GF_XOR : 0); + UPTR_CAST(x,32)[3] = ((UPTR_CAST(x,32)[3] << 1) & ~MASK_01) + | (((UPTR_CAST(x,32)[3] >> 15) | (UPTR_CAST(x,32)[2] << 17)) & MASK_01); + UPTR_CAST(x,32)[2] = ((UPTR_CAST(x,32)[2] << 1) & ~MASK_01) + | (((UPTR_CAST(x,32)[2] >> 15) | (UPTR_CAST(x,32)[1] << 17)) & MASK_01); + UPTR_CAST(x,32)[1] = ((UPTR_CAST(x,32)[1] << 1) & ~MASK_01) + | (((UPTR_CAST(x,32)[1] >> 15) | (UPTR_CAST(x,32)[0] << 17)) & MASK_01); + UPTR_CAST(x,32)[0] = (((UPTR_CAST(x,32)[0] << 1) & ~MASK_01) + | ((UPTR_CAST(x,32)[0] >> 15) & MASK_01)) ^ _tt; + +# endif + +#endif +} + +INT_RETURN xts_encrypt_key( const unsigned char key[], int key_len, xts_encrypt_ctx ctx[1] ) +{ int aes_klen_by; + + switch( key_len ) + { + default: return EXIT_FAILURE; + case 32: + case 256: aes_klen_by = 16; break; + case 64: + case 512: aes_klen_by = 32; break; + } + + rijndael_set_key(ctx->enc_ctx, UPTR_CAST(key, 32), aes_klen_by*8, 1); + rijndael_set_key(ctx->twk_ctx, UPTR_CAST(key + aes_klen_by, 32), aes_klen_by*8, 1); + return EXIT_SUCCESS; +} + +INT_RETURN xts_decrypt_key( const unsigned char key[], int key_len, xts_decrypt_ctx ctx[1] ) +{ int aes_klen_by; + + switch( key_len ) + { + default: return EXIT_FAILURE; + case 32: + case 256: aes_klen_by = 16; break; + case 64: + case 512: aes_klen_by = 32; break; + } + + rijndael_set_key(ctx->dec_ctx, UPTR_CAST(key, 32), aes_klen_by*8, 0); + rijndael_set_key(ctx->twk_ctx, UPTR_CAST(key + aes_klen_by, 32), aes_klen_by*8, 1); + return EXIT_SUCCESS; +} + +INT_RETURN xts_encrypt_block( unsigned char sector[], const unsigned char tweak[], + unsigned int sector_len, xts_encrypt_ctx ctx[1] ) +{ + buf_type hh; + uint_8t *pos = sector, *hi = sector + sector_len; + + xor_function f_ptr = (!ALIGN_OFFSET(sector, UNIT_BITS >> 3) ? xor_block_aligned : xor_block ); + + if( sector_len < AES_BLOCK_SIZE ) + return EXIT_FAILURE; + + rijndael_encrypt(ctx->twk_ctx, UPTR_CAST(tweak, 32), UPTR_CAST(hh, 32)); + + while(pos + AES_BLOCK_SIZE <= hi) + { + f_ptr(pos, pos, hh); + rijndael_encrypt(ctx->enc_ctx, UPTR_CAST(pos, 32), UPTR_CAST(pos, 32)); + f_ptr(pos, pos, hh); + pos += AES_BLOCK_SIZE; + gf_mulx(hh); + } + + if(pos < hi) + { + uint_8t *tp = pos - AES_BLOCK_SIZE; + while(pos < hi) + { + uint_8t tt = *(pos - AES_BLOCK_SIZE); + *(pos - AES_BLOCK_SIZE) = *pos; + *pos++ = tt; + } + f_ptr(tp, tp, hh); + rijndael_encrypt(ctx->enc_ctx, UPTR_CAST(tp, 32), UPTR_CAST(tp, 32)); + f_ptr(tp, tp, hh); + } + return EXIT_SUCCESS; +} + +INT_RETURN xts_decrypt_block( unsigned char sector[], const unsigned char tweak[], + unsigned int sector_len, xts_decrypt_ctx ctx[1] ) +{ + buf_type hh, hh2; + uint_8t *pos = sector, *hi = sector + sector_len; + + xor_function f_ptr = (!ALIGN_OFFSET(sector, UNIT_BITS >> 3) ? xor_block_aligned : xor_block ); + + if( sector_len < AES_BLOCK_SIZE ) + return EXIT_FAILURE; + + rijndael_encrypt(ctx->twk_ctx, UPTR_CAST(tweak, 32), UPTR_CAST(hh, 32)); + + while(pos + AES_BLOCK_SIZE <= hi) + { + if(hi - pos > AES_BLOCK_SIZE && hi - pos < 2 * AES_BLOCK_SIZE) + { + memcpy(hh2, hh, AES_BLOCK_SIZE); + gf_mulx(hh); + } + f_ptr(pos, pos, hh); + rijndael_decrypt(ctx->dec_ctx, UPTR_CAST(pos, 32), UPTR_CAST(pos, 32)); + f_ptr(pos, pos, hh); + pos += AES_BLOCK_SIZE; + gf_mulx(hh); + } + + if(pos < hi) + { + uint_8t *tp = pos - AES_BLOCK_SIZE; + while(pos < hi) + { + uint_8t tt = *(pos - AES_BLOCK_SIZE); + *(pos - AES_BLOCK_SIZE) = *pos; + *pos++ = tt; + } + f_ptr(tp, tp, hh2); + rijndael_decrypt(ctx->dec_ctx, UPTR_CAST(tp, 32), UPTR_CAST(tp, 32)); + f_ptr(tp, tp, hh2); + } + + return EXIT_SUCCESS; +} + diff --git a/contrib/pgcrypto/xts.h b/contrib/pgcrypto/xts.h new file mode 100644 index 0000000..88b2b7b --- /dev/null +++ b/contrib/pgcrypto/xts.h @@ -0,0 +1,62 @@ +/* + --------------------------------------------------------------------------- + Copyright (c) 1998-2008, Brian Gladman, Worcester, UK. All rights reserved. + + LICENSE TERMS + + The redistribution and use of this software (with or without changes) + is allowed without the payment of fees or royalties provided that: + + 1. source code distributions include the above copyright notice, this + list of conditions and the following disclaimer; + + 2. binary distributions include the above copyright notice, this list + of conditions and the following disclaimer in their documentation; + + 3. the name of the copyright holder is not used to endorse products + built using this software without specific written permission. + + DISCLAIMER + + This software is provided 'as is' with no explicit or implied warranties + in respect of its properties, including, but not limited to, correctness + and/or fitness for purpose. + --------------------------------------------------------------------------- + Issue Date: 20/12/2007 +*/ + +#ifndef _XTS_H +#define _XTS_H + +#include "rijndael.h" + +/* lifted from aes.h */ +#define AES_BLOCK_SIZE 16 /* the AES block size in bytes */ + +/* end */ + +typedef struct +{ + rijndael_ctx twk_ctx[1]; + rijndael_ctx enc_ctx[1]; +} xts_encrypt_ctx; + +typedef struct +{ + rijndael_ctx twk_ctx[1]; + rijndael_ctx dec_ctx[1]; +} xts_decrypt_ctx; + +#define INT_RETURN int + +INT_RETURN xts_encrypt_key( const unsigned char key[], int key_len, xts_encrypt_ctx ctx[1] ); + +INT_RETURN xts_decrypt_key( const unsigned char key[], int key_len, xts_decrypt_ctx ctx[1] ); + +INT_RETURN xts_encrypt_block( unsigned char sector[], const unsigned char tweak[], + unsigned int sector_len, xts_encrypt_ctx ctx[1] ); + +INT_RETURN xts_decrypt_block( unsigned char sector[], const unsigned char tweak[], + unsigned int sector_len, xts_decrypt_ctx ctx[1] ); + +#endif diff --git a/src/backend/access/transam/slru.c b/src/backend/access/transam/slru.c index 36a011c..43da67d 100644 --- a/src/backend/access/transam/slru.c +++ b/src/backend/access/transam/slru.c @@ -54,6 +54,7 @@ #include "access/slru.h" #include "access/transam.h" #include "access/xlog.h" +#include "storage/encryption.h" #include "storage/fd.h" #include "storage/shmem.h" #include "miscadmin.h" @@ -121,6 +122,8 @@ typedef enum static SlruErrorCause slru_errcause; static int slru_errno; +static char slru_encryption_buf[BLCKSZ]; +static char slru_encryption_tweak[TWEAK_SIZE]; static void SimpleLruZeroLSNs(SlruCtl ctl, int slotno); @@ -135,6 +138,7 @@ static int SlruSelectLRUPage(SlruCtl ctl, int pageno); static bool SlruScanDirCbDeleteCutoff(SlruCtl ctl, char *filename, int segpage, void *data); static void SlruInternalDeleteSegment(SlruCtl ctl, char *filename); +static void SlruEncryptionTweak(char *tweak, int pageno); /* * Initialization of shared memory @@ -641,6 +645,7 @@ SlruPhysicalReadPage(SlruCtl ctl, int pageno, int slotno) int offset = rpageno * BLCKSZ; char path[MAXPGPATH]; int fd; + char *rbuf; SlruFileName(ctl, path, segno); @@ -676,8 +681,13 @@ SlruPhysicalReadPage(SlruCtl ctl, int pageno, int slotno) return false; } + if (encryption_enabled) + rbuf = slru_encryption_buf; + else + rbuf = shared->page_buffer[slotno]; + errno = 0; - if (read(fd, shared->page_buffer[slotno], BLCKSZ) != BLCKSZ) + if (read(fd, rbuf, BLCKSZ) != BLCKSZ) { slru_errcause = SLRU_READ_FAILED; slru_errno = errno; @@ -685,6 +695,14 @@ SlruPhysicalReadPage(SlruCtl ctl, int pageno, int slotno) return false; } + + if (encryption_enabled) + { + SlruEncryptionTweak(slru_encryption_tweak, pageno); + decrypt_block(slru_encryption_buf, shared->page_buffer[slotno], + BLCKSZ, slru_encryption_tweak); + } + if (CloseTransientFile(fd)) { slru_errcause = SLRU_CLOSE_FAILED; @@ -718,6 +736,7 @@ SlruPhysicalWritePage(SlruCtl ctl, int pageno, int slotno, SlruFlush fdata) int offset = rpageno * BLCKSZ; char path[MAXPGPATH]; int fd = -1; + char *wbuf; /* * Honor the write-WAL-before-data rule, if appropriate, so that we do not @@ -835,8 +854,16 @@ SlruPhysicalWritePage(SlruCtl ctl, int pageno, int slotno, SlruFlush fdata) return false; } + wbuf = shared->page_buffer[slotno]; + if (encryption_enabled) + { + SlruEncryptionTweak(slru_encryption_tweak, pageno); + encrypt_block(wbuf, slru_encryption_buf, BLCKSZ, slru_encryption_tweak); + wbuf = slru_encryption_buf; + } + errno = 0; - if (write(fd, shared->page_buffer[slotno], BLCKSZ) != BLCKSZ) + if (write(fd, wbuf, BLCKSZ) != BLCKSZ) { /* if write didn't set errno, assume problem is no disk space */ if (errno == 0) @@ -1392,3 +1419,14 @@ SlruScanDirectory(SlruCtl ctl, SlruScanCallback callback, void *data) return retval; } + +/* + * SLRU data encryption is tweaked by page number. + */ +static void +SlruEncryptionTweak(char *tweak, int pageno) +{ + /* TODO: would be nice to incorporate SLRU type in tweak */ + memcpy(tweak, &pageno, sizeof(pageno)); + memset(tweak + sizeof(pageno), 0, TWEAK_SIZE - sizeof(pageno)); +} diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index b473f19..640374d 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -53,6 +53,7 @@ #include "replication/walsender.h" #include "storage/barrier.h" #include "storage/bufmgr.h" +#include "storage/encryption.h" #include "storage/fd.h" #include "storage/ipc.h" #include "storage/large_object.h" @@ -74,6 +75,8 @@ #include "pg_trace.h" extern uint32 bootstrap_data_checksum_version; +extern bool bootstrap_data_encrypted; +extern char *bootstrap_encryption_sample; /* File path names (all relative to $PGDATA) */ #define RECOVERY_COMMAND_FILE "recovery.conf" @@ -803,6 +806,7 @@ static void LocalSetXLogInsertAllowed(void); static void CreateEndOfRecoveryRecord(void); static void CheckPointGuts(XLogRecPtr checkPointRedo, int flags); static void KeepLogSeg(XLogRecPtr recptr, XLogSegNo *logSegNo); +static void XLogWritePages(char *from, int npages); static XLogRecPtr XLogGetReplicationSlotMinimumLSN(void); static void AdvanceXLInsertBuffer(XLogRecPtr upto, bool opportunistic); @@ -2260,8 +2264,6 @@ XLogWrite(XLogwrtRqst WriteRqst, bool flexible) { char *from; Size nbytes; - Size nleft; - int written; /* Need to seek in the file? */ if (openLogOff != startoffset) @@ -2278,28 +2280,28 @@ XLogWrite(XLogwrtRqst WriteRqst, bool flexible) /* OK to write the page(s) */ from = XLogCtl->pages + startidx * (Size) XLOG_BLCKSZ; nbytes = npages * (Size) XLOG_BLCKSZ; - nleft = nbytes; - do - { - errno = 0; - written = write(openLogFile, from, nleft); - if (written <= 0) - { - if (errno == EINTR) - continue; - ereport(PANIC, - (errcode_for_file_access(), - errmsg("could not write to log file %s " - "at offset %u, length %zu: %m", - XLogFileNameP(ThisTimeLineID, openLogSegNo), - openLogOff, nbytes))); - } - nleft -= written; - from += written; - } while (nleft > 0); + if (encryption_enabled) { + int i; + /* + * XXX: use larger encryption buffer to enable larger writes + * and reduce number of syscalls? + */ + for (i = 0; i < npages; i++) { + char buf[XLOG_BLCKSZ]; + char tweak[TWEAK_SIZE]; + XLogEncryptionTweak(tweak, ThisTimeLineID, openLogSegNo, openLogOff); + encrypt_block(from, buf, XLOG_BLCKSZ, tweak); + XLogWritePages(buf, 1); + + from += XLOG_BLCKSZ; + openLogOff += XLOG_BLCKSZ; + } + } else { + XLogWritePages(from, npages); + openLogOff += nbytes; + } /* Update state for write */ - openLogOff += nbytes; npages = 0; /* @@ -2412,6 +2414,32 @@ XLogWrite(XLogwrtRqst WriteRqst, bool flexible) } } +static void +XLogWritePages(char *from, int npages) +{ + Size nleft = npages * (Size) XLOG_BLCKSZ; + Size written; + + do + { + errno = 0; + written = write(openLogFile, from, nleft); + if (written <= 0) + { + if (errno == EINTR) + continue; + ereport(PANIC, + (errcode_for_file_access(), + errmsg("could not write to log file %s " + "at offset %u, length %zu: %m", + XLogFileNameP(ThisTimeLineID, openLogSegNo), + openLogOff, npages * (Size) XLOG_BLCKSZ))); + } + nleft -= written; + from += written; + } while (nleft > 0); +} + /* * Record the LSN for an asynchronous transaction commit/abort * and nudge the WALWriter if there is work for it to do. @@ -4454,6 +4482,24 @@ ReadControlFile(void) errhint("It looks like you need to recompile or initdb."))); #endif + if (ControlFile->data_encrypted && !encryption_enabled) + ereport(FATAL, + (errmsg("database files are encrypted"), + errdetail("The database cluster was initialized with encryption" + " but the server was started without an encryption module."), + errhint("Set the encryption module using " + "encryption_library configuration parameter."))); + else if (encryption_enabled) + { + char sample[ENCRYPTION_SAMPLE_SIZE]; + sample_encryption(sample); + if (memcmp(ControlFile->encryption_verification, sample, ENCRYPTION_SAMPLE_SIZE)) + ereport(FATAL, + (errmsg("invalid encryption key"), + errdetail("The key specified in PGENCRYPTIONKEY does not match" + " database encryption key."))); + } + /* Make the initdb settings visible as GUC variables, too */ SetConfigOption("data_checksums", DataChecksumsEnabled() ? "yes" : "no", PGC_INTERNAL, PGC_S_OVERRIDE); @@ -4863,6 +4909,13 @@ BootStrapXLOG(void) use_existent = false; openLogFile = XLogFileInit(1, &use_existent, false); + if (encryption_enabled) + { + char tweak[TWEAK_SIZE]; + XLogEncryptionTweak(tweak, ThisTimeLineID, 1, 0); + encrypt_block((char*)page, (char*)page, XLOG_BLCKSZ, tweak); + } + /* Write the first page with the initial record */ errno = 0; if (write(openLogFile, page, XLOG_BLCKSZ) != XLOG_BLCKSZ) @@ -4907,6 +4960,11 @@ BootStrapXLOG(void) ControlFile->wal_log_hints = wal_log_hints; ControlFile->track_commit_timestamp = track_commit_timestamp; ControlFile->data_checksum_version = bootstrap_data_checksum_version; + ControlFile->data_encrypted = bootstrap_data_encrypted; + if (bootstrap_data_encrypted) + memcpy(ControlFile->encryption_verification, bootstrap_encryption_sample, 16); + else + memset(ControlFile->encryption_verification, 0, 16); /* some additional ControlFile fields are set in WriteControlFile() */ @@ -11080,6 +11138,13 @@ retry: Assert(targetPageOff == readOff); Assert(reqLen <= readLen); + if (encryption_enabled) + { + char tweak[TWEAK_SIZE]; + XLogEncryptionTweak(tweak, curFileTLI, readSegNo, readOff); + decrypt_block(readBuf, readBuf, XLOG_BLCKSZ, tweak); + } + *readTLI = curFileTLI; return readLen; diff --git a/src/backend/access/transam/xlogutils.c b/src/backend/access/transam/xlogutils.c index 51a8e8d..1f47bba 100644 --- a/src/backend/access/transam/xlogutils.c +++ b/src/backend/access/transam/xlogutils.c @@ -24,6 +24,7 @@ #include "access/xlogutils.h" #include "catalog/catalog.h" #include "miscadmin.h" +#include "storage/encryption.h" #include "storage/smgr.h" #include "utils/guc.h" #include "utils/hsearch.h" @@ -653,18 +654,24 @@ XLogTruncateRelation(RelFileNode rnode, ForkNumber forkNum, static void XLogRead(char *buf, TimeLineID tli, XLogRecPtr startptr, Size count) { - char *p; + char *p, *decrypt_p; XLogRecPtr recptr; Size nbytes; + uint32 decryptOff; /* state maintained across calls */ static int sendFile = -1; static XLogSegNo sendSegNo = 0; static uint32 sendOff = 0; - p = buf; + /* We only support block aligned reads to support encryption */ + Assert(startptr % XLOG_BLCKSZ == 0); + Assert(count % XLOG_BLCKSZ == 0); + + decrypt_p = p = buf; recptr = startptr; nbytes = count; + decryptOff = startptr % XLogSegSize; while (nbytes > 0) { @@ -746,6 +753,20 @@ XLogRead(char *buf, TimeLineID tli, XLogRecPtr startptr, Size count) sendOff += readbytes; nbytes -= readbytes; p += readbytes; + + /* Decrypt completed blocks */ + if (encryption_enabled) + { + while (decrypt_p + XLOG_BLCKSZ <= p) + { + char tweak[TWEAK_SIZE]; + XLogEncryptionTweak(tweak, tli, sendSegNo, decryptOff); + decrypt_block(decrypt_p, decrypt_p, XLOG_BLCKSZ, tweak); + + decrypt_p += XLOG_BLCKSZ; + decryptOff += XLOG_BLCKSZ; + } + } } } @@ -823,3 +844,15 @@ read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, /* number of valid bytes in the buffer */ return count; } + +/* + * Xlog is encrypted page at a time. Each xlog page gets a unique tweak via + * timeline, segment and offset. + */ +void +XLogEncryptionTweak(char *tweak, TimeLineID timeline, XLogSegNo segment, uint32 offset) +{ + memcpy(tweak, &segment, sizeof(XLogSegNo)); + memcpy(tweak + sizeof(XLogSegNo), &offset, sizeof(offset)); + memcpy(tweak + sizeof(XLogSegNo) + sizeof(uint32), &timeline, sizeof(timeline)); +} diff --git a/src/backend/bootstrap/bootstrap.c b/src/backend/bootstrap/bootstrap.c index e518e17..ab2a8e2 100644 --- a/src/backend/bootstrap/bootstrap.c +++ b/src/backend/bootstrap/bootstrap.c @@ -21,6 +21,7 @@ #include "bootstrap/bootstrap.h" #include "catalog/index.h" #include "catalog/pg_collation.h" +#include "catalog/pg_control.h" #include "catalog/pg_type.h" #include "libpq/pqsignal.h" #include "miscadmin.h" @@ -33,6 +34,7 @@ #include "replication/walreceiver.h" #include "storage/bufmgr.h" #include "storage/bufpage.h" +#include "storage/encryption.h" #include "storage/ipc.h" #include "storage/proc.h" #include "tcop/tcopprot.h" @@ -45,7 +47,8 @@ #include "utils/tqual.h" uint32 bootstrap_data_checksum_version = 0; /* No checksum */ - +bool bootstrap_data_encrypted = false; +char *bootstrap_encryption_sample = NULL; #define ALLOC(t, c) ((t *) calloc((unsigned)(c), sizeof(t))) @@ -352,6 +355,17 @@ AuxiliaryProcessMain(int argc, char *argv[]) if (!IsUnderPostmaster) InitializeMaxBackends(); + if (!IsUnderPostmaster) + setup_encryption(); + + if (encryption_enabled) + { + bootstrap_data_encrypted = true; + bootstrap_encryption_sample = palloc0(ENCRYPTION_SAMPLE_SIZE); + sample_encryption(bootstrap_encryption_sample); + } + + BaseInit(); /* diff --git a/src/backend/commands/dbcommands.c b/src/backend/commands/dbcommands.c index c1c0223..d7054bc 100644 --- a/src/backend/commands/dbcommands.c +++ b/src/backend/commands/dbcommands.c @@ -615,7 +615,11 @@ createdb(const CreatedbStmt *stmt) * * We don't need to copy subdirectories */ - copydir(srcpath, dstpath, false); + { + RelFileNode fromNode = {srctablespace, src_dboid, InvalidOid}; + RelFileNode toNode = {dsttablespace, dboid, InvalidOid}; + copydir(srcpath, dstpath, &fromNode, &toNode); + } /* Record the filesystem change in XLOG */ { @@ -1221,7 +1225,11 @@ movedb(const char *dbname, const char *tblspcname) /* * Copy files from the old tablespace to the new one */ - copydir(src_dbpath, dst_dbpath, false); + { + RelFileNode fromNode = {src_tblspcoid, db_id, InvalidOid}; + RelFileNode toNode = {dst_tblspcoid, db_id, InvalidOid}; + copydir(src_dbpath, dst_dbpath, &fromNode, &toNode); + } /* * Record the filesystem change in XLOG @@ -2084,7 +2092,11 @@ dbase_redo(XLogReaderState *record) * * We don't need to copy subdirectories */ - copydir(src_path, dst_path, false); + { + RelFileNode fromNode = {xlrec->src_tablespace_id, xlrec->src_db_id, InvalidOid}; + RelFileNode toNode = {xlrec->tablespace_id, xlrec->db_id, InvalidOid}; + copydir(src_path, dst_path, &fromNode, &toNode); + } } else if (info == XLOG_DBASE_DROP) { diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c index 6cf51e1..c16f11c 100644 --- a/src/backend/postmaster/postmaster.c +++ b/src/backend/postmaster/postmaster.c @@ -114,6 +114,7 @@ #include "postmaster/postmaster.h" #include "postmaster/syslogger.h" #include "replication/walsender.h" +#include "storage/encryption.h" #include "storage/fd.h" #include "storage/ipc.h" #include "storage/pg_shmem.h" @@ -920,6 +921,8 @@ PostmasterMain(int argc, char *argv[]) secure_initialize(); #endif + setup_encryption(); + /* * process any libraries that should be preloaded at postmaster start */ diff --git a/src/backend/storage/file/copydir.c b/src/backend/storage/file/copydir.c index a51ee81..4911c51 100644 --- a/src/backend/storage/file/copydir.c +++ b/src/backend/storage/file/copydir.c @@ -22,25 +22,31 @@ #include #include +#include "catalog/catalog.h" #include "storage/copydir.h" +#include "storage/encryption.h" #include "storage/fd.h" +#include "storage/smgr.h" #include "miscadmin.h" /* * copydir: copy a directory * - * If recurse is false, subdirectories are ignored. Anything that's not - * a directory or a regular file is ignored. + * RelFileNode values must specify tablespace and database oids for source + * and target to support re-encryption if necessary. relNode value in provided + * structs will be clobbered. */ void -copydir(char *fromdir, char *todir, bool recurse) +copydir(char *fromdir, char *todir, RelFileNode *fromNode, RelFileNode *toNode) { DIR *xldir; struct dirent *xlde; char fromfile[MAXPGPATH]; char tofile[MAXPGPATH]; + Assert(!encryption_enabled || (fromNode != NULL && toNode != NULL)); + if (mkdir(todir, S_IRWXU) != 0) ereport(ERROR, (errcode_for_file_access(), @@ -71,14 +77,31 @@ copydir(char *fromdir, char *todir, bool recurse) (errcode_for_file_access(), errmsg("could not stat file \"%s\": %m", fromfile))); - if (S_ISDIR(fst.st_mode)) + if (S_ISREG(fst.st_mode)) { - /* recurse to handle subdirectories */ - if (recurse) - copydir(fromfile, tofile, true); + int oidchars; + ForkNumber forkNum; + int segment; + + /* + * For encrypted databases we need to reencrypt files with new + * tweaks. + */ + if (encryption_enabled && + parse_filename_for_nontemp_relation(xlde->d_name, + &oidchars, &forkNum, &segment)) + { + char oidbuf[OIDCHARS+1]; + memcpy(oidbuf, xlde->d_name, oidchars); + oidbuf[oidchars] = '\0'; + + /* We scribble over the provided RelFileNodes here */ + fromNode->relNode = toNode->relNode = atol(oidbuf); + copy_file(fromfile, tofile, fromNode, toNode, forkNum, forkNum, segment); + } + else + copy_file(fromfile, tofile, NULL, NULL, 0, 0, 0); } - else if (S_ISREG(fst.st_mode)) - copy_file(fromfile, tofile); } FreeDir(xldir); @@ -129,15 +152,20 @@ copydir(char *fromdir, char *todir, bool recurse) } /* - * copy one file + * copy one file. If decryption and reencryption is needed specify + * relfilenodes for source and target. */ void -copy_file(char *fromfile, char *tofile) +copy_file(char *fromfile, char *tofile, RelFileNode *fromNode, + RelFileNode *toNode, ForkNumber fromForkNum, ForkNumber toForkNum, + int segment) { char *buffer; int srcfd; int dstfd; int nbytes; + int bytesread; + BlockNumber blockNum = segment*RELSEG_SIZE; off_t offset; /* Use palloc to ensure we get a maxaligned buffer */ @@ -169,14 +197,37 @@ copy_file(char *fromfile, char *tofile) /* If we got a cancel signal during the copy of the file, quit */ CHECK_FOR_INTERRUPTS(); - nbytes = read(srcfd, buffer, COPY_BUF_SIZE); - if (nbytes < 0) - ereport(ERROR, - (errcode_for_file_access(), - errmsg("could not read file \"%s\": %m", fromfile))); + /* + * Try to read as much as we fit in the buffer so we can deal with + * complete blocks if we need to reencrypt. + */ + nbytes = 0; + while (nbytes < COPY_BUF_SIZE) + { + bytesread = read(srcfd, buffer, COPY_BUF_SIZE); + if (bytesread < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not read file \"%s\": %m", fromfile))); + nbytes += bytesread; + if (bytesread == 0) + break; + } if (nbytes == 0) break; errno = 0; + + /* + * If the database is encrypted we need to decrypt the data here + * and reencrypt it to adjust the tweak values of blocks. + */ + if (fromNode != NULL) + { + Assert(toNode != NULL); + blockNum = ReencryptBlock(buffer, nbytes/BLCKSZ, + fromNode, toNode, fromForkNum, toForkNum, blockNum); + } + if ((int) write(dstfd, buffer, nbytes) != nbytes) { /* if write didn't set errno, assume problem is no disk space */ @@ -204,3 +255,71 @@ copy_file(char *fromfile, char *tofile) pfree(buffer); } + + +/* + * Basic parsing of putative relation filenames. + * + * This function returns true if the file appears to be in the correct format + * for a non-temporary relation and false otherwise. + * + * NB: If this function returns true, the caller is entitled to assume that + * *oidchars has been set to the a value no more than OIDCHARS, and thus + * that a buffer of OIDCHARS+1 characters is sufficient to hold the OID + * portion of the filename. This is critical to protect against a possible + * buffer overrun. + */ +bool +parse_filename_for_nontemp_relation(const char *name, int *oidchars, + ForkNumber *fork, int *segment) +{ + int pos; + int segstart = 0; + + /* Look for a non-empty string of digits (that isn't too long). */ + for (pos = 0; isdigit((unsigned char) name[pos]); ++pos) + ; + if (pos == 0 || pos > OIDCHARS) + return false; + *oidchars = pos; + + /* Check for a fork name. */ + if (name[pos] != '_') + *fork = MAIN_FORKNUM; + else + { + int forkchar; + + forkchar = forkname_chars(&name[pos + 1], fork); + if (forkchar <= 0) + return false; + pos += forkchar + 1; + } + + /* Check for a segment number. */ + if (name[pos] == '.') + { + int segchar; + + segstart = pos + 1; + for (segchar = 1; isdigit((unsigned char) name[pos + segchar]); ++segchar) + ; + if (segchar <= 1) + return false; + pos += segchar; + } + + /* Now we should be at the end. */ + if (name[pos] != '\0') + return false; + + if (segment != NULL) + { + if (segstart == 0) + *segment = 0; + else + *segment = atoi(name + segstart); + } + + return true; +} diff --git a/src/backend/storage/file/reinit.c b/src/backend/storage/file/reinit.c index 7e8138b..326036c 100644 --- a/src/backend/storage/file/reinit.c +++ b/src/backend/storage/file/reinit.c @@ -17,6 +17,7 @@ #include #include "catalog/catalog.h" +#include "catalog/pg_tablespace.h" #include "common/relpath.h" #include "storage/copydir.h" #include "storage/fd.h" @@ -25,11 +26,9 @@ #include "utils/memutils.h" static void ResetUnloggedRelationsInTablespaceDir(const char *tsdirname, - int op); + int op, Oid spcOid); static void ResetUnloggedRelationsInDbspaceDir(const char *dbspacedirname, - int op); -static bool parse_filename_for_nontemp_relation(const char *name, - int *oidchars, ForkNumber *fork); + int op, Oid spcOid, Oid dbOid); typedef struct { @@ -73,7 +72,7 @@ ResetUnloggedRelations(int op) /* * First process unlogged files in pg_default ($PGDATA/base) */ - ResetUnloggedRelationsInTablespaceDir("base", op); + ResetUnloggedRelationsInTablespaceDir("base", op, DEFAULTTABLESPACE_OID); /* * Cycle through directories for all non-default tablespaces. @@ -82,13 +81,15 @@ ResetUnloggedRelations(int op) while ((spc_de = ReadDir(spc_dir, "pg_tblspc")) != NULL) { + Oid spcOid; if (strcmp(spc_de->d_name, ".") == 0 || strcmp(spc_de->d_name, "..") == 0) continue; snprintf(temp_path, sizeof(temp_path), "pg_tblspc/%s/%s", spc_de->d_name, TABLESPACE_VERSION_DIRECTORY); - ResetUnloggedRelationsInTablespaceDir(temp_path, op); + spcOid = atoi(spc_de->d_name); + ResetUnloggedRelationsInTablespaceDir(temp_path, op, spcOid); } FreeDir(spc_dir); @@ -102,7 +103,7 @@ ResetUnloggedRelations(int op) /* Process one tablespace directory for ResetUnloggedRelations */ static void -ResetUnloggedRelationsInTablespaceDir(const char *tsdirname, int op) +ResetUnloggedRelationsInTablespaceDir(const char *tsdirname, int op, Oid spcOid) { DIR *ts_dir; struct dirent *de; @@ -122,6 +123,7 @@ ResetUnloggedRelationsInTablespaceDir(const char *tsdirname, int op) while ((de = ReadDir(ts_dir, tsdirname)) != NULL) { int i = 0; + Oid dbOid; /* * We're only interested in the per-database directories, which have @@ -133,9 +135,10 @@ ResetUnloggedRelationsInTablespaceDir(const char *tsdirname, int op) if (de->d_name[i] != '\0' || i == 0) continue; + dbOid = atoi(de->d_name); snprintf(dbspace_path, sizeof(dbspace_path), "%s/%s", tsdirname, de->d_name); - ResetUnloggedRelationsInDbspaceDir(dbspace_path, op); + ResetUnloggedRelationsInDbspaceDir(dbspace_path, op, spcOid, dbOid); } FreeDir(ts_dir); @@ -143,7 +146,8 @@ ResetUnloggedRelationsInTablespaceDir(const char *tsdirname, int op) /* Process one per-dbspace directory for ResetUnloggedRelations */ static void -ResetUnloggedRelationsInDbspaceDir(const char *dbspacedirname, int op) +ResetUnloggedRelationsInDbspaceDir(const char *dbspacedirname, int op, + Oid spcOid, Oid dbOid) { DIR *dbspace_dir; struct dirent *de; @@ -192,7 +196,7 @@ ResetUnloggedRelationsInDbspaceDir(const char *dbspacedirname, int op) /* Skip anything that doesn't look like a relation data file. */ if (!parse_filename_for_nontemp_relation(de->d_name, &oidchars, - &forkNum)) + &forkNum, NULL)) continue; /* Also skip it unless this is the init fork. */ @@ -245,7 +249,7 @@ ResetUnloggedRelationsInDbspaceDir(const char *dbspacedirname, int op) /* Skip anything that doesn't look like a relation data file. */ if (!parse_filename_for_nontemp_relation(de->d_name, &oidchars, - &forkNum)) + &forkNum, NULL)) continue; /* We never remove the init fork. */ @@ -309,13 +313,14 @@ ResetUnloggedRelationsInDbspaceDir(const char *dbspacedirname, int op) { ForkNumber forkNum; int oidchars; + int segment; char oidbuf[OIDCHARS + 1]; char srcpath[MAXPGPATH]; char dstpath[MAXPGPATH]; /* Skip anything that doesn't look like a relation data file. */ if (!parse_filename_for_nontemp_relation(de->d_name, &oidchars, - &forkNum)) + &forkNum, &segment)) continue; /* Also skip it unless this is the init fork. */ @@ -335,7 +340,12 @@ ResetUnloggedRelationsInDbspaceDir(const char *dbspacedirname, int op) /* OK, we're ready to perform the actual copy. */ elog(DEBUG2, "copying %s to %s", srcpath, dstpath); - copy_file(srcpath, dstpath); + { + RelFileNode srcNode = {spcOid, dbOid, atol(oidbuf)}; + RelFileNode dstNode = srcNode; + copy_file(srcpath, dstpath, &srcNode, &dstNode, + INIT_FORKNUM, MAIN_FORKNUM, segment); + } } FreeDir(dbspace_dir); @@ -366,7 +376,7 @@ ResetUnloggedRelationsInDbspaceDir(const char *dbspacedirname, int op) /* Skip anything that doesn't look like a relation data file. */ if (!parse_filename_for_nontemp_relation(de->d_name, &oidchars, - &forkNum)) + &forkNum, NULL)) continue; /* Also skip it unless this is the init fork. */ @@ -388,59 +398,3 @@ ResetUnloggedRelationsInDbspaceDir(const char *dbspacedirname, int op) fsync_fname(dbspacedirname, true); } } - -/* - * Basic parsing of putative relation filenames. - * - * This function returns true if the file appears to be in the correct format - * for a non-temporary relation and false otherwise. - * - * NB: If this function returns true, the caller is entitled to assume that - * *oidchars has been set to the a value no more than OIDCHARS, and thus - * that a buffer of OIDCHARS+1 characters is sufficient to hold the OID - * portion of the filename. This is critical to protect against a possible - * buffer overrun. - */ -static bool -parse_filename_for_nontemp_relation(const char *name, int *oidchars, - ForkNumber *fork) -{ - int pos; - - /* Look for a non-empty string of digits (that isn't too long). */ - for (pos = 0; isdigit((unsigned char) name[pos]); ++pos) - ; - if (pos == 0 || pos > OIDCHARS) - return false; - *oidchars = pos; - - /* Check for a fork name. */ - if (name[pos] != '_') - *fork = MAIN_FORKNUM; - else - { - int forkchar; - - forkchar = forkname_chars(&name[pos + 1], fork); - if (forkchar <= 0) - return false; - pos += forkchar + 1; - } - - /* Check for a segment number. */ - if (name[pos] == '.') - { - int segchar; - - for (segchar = 1; isdigit((unsigned char) name[pos + segchar]); ++segchar) - ; - if (segchar <= 1) - return false; - pos += segchar; - } - - /* Now we should be at the end. */ - if (name[pos] != '\0') - return false; - return true; -} diff --git a/src/backend/storage/page/bufpage.c b/src/backend/storage/page/bufpage.c index f2a07f2..423034e 100644 --- a/src/backend/storage/page/bufpage.c +++ b/src/backend/storage/page/bufpage.c @@ -119,18 +119,7 @@ PageIsVerified(Page page, BlockNumber blkno) } /* Check all-zeroes case */ - all_zeroes = true; - pagebytes = (char *) page; - for (i = 0; i < BLCKSZ; i++) - { - if (pagebytes[i] != 0) - { - all_zeroes = false; - break; - } - } - - if (all_zeroes) + if (IsAllZero((char *) page, BLCKSZ)) return true; /* @@ -1134,3 +1123,38 @@ PageSetChecksumInplace(Page page, BlockNumber blkno) ((PageHeader) page)->pd_checksum = pg_checksum_page((char *) page, blkno); } + +/* + * Helper function to check if a page is completely empty. + */ +bool +IsAllZero(const char *input, Size size) +{ + const char *pos = input; + const char *aligned_start = (char*) MAXALIGN64(input); + const char *end = input + size; + + /* Check 1 byte at a time until pos is 8 byte aligned */ + while (pos < aligned_start) + if (*pos++ != 0) + return false; + + /* + * Run 8 parallel 8 byte checks in one iteration. On 2016 hardware + * slightly faster than 4 parallel checks. + **/ + while (pos + 8*sizeof(uint64) <= end) + { + uint64 *p = (uint64*) pos; + if ((p[0] | p[1] | p[2] | p[3] | p[4] | p[5] | p[6] | p[7]) != 0) + return false; + pos += 8*sizeof(uint64); + } + + /* Handle unaligned tail. */ + while (pos < end) + if (*pos++ != 0) + return false; + + return true; +} diff --git a/src/backend/storage/smgr/Makefile b/src/backend/storage/smgr/Makefile index 2b95cb0..0601ea7 100644 --- a/src/backend/storage/smgr/Makefile +++ b/src/backend/storage/smgr/Makefile @@ -12,6 +12,6 @@ subdir = src/backend/storage/smgr top_builddir = ../../../.. include $(top_builddir)/src/Makefile.global -OBJS = md.o smgr.o smgrtype.o +OBJS = encryption.o md.o smgr.o smgrtype.o include $(top_srcdir)/src/backend/common.mk diff --git a/src/backend/storage/smgr/encryption.c b/src/backend/storage/smgr/encryption.c new file mode 100644 index 0000000..766487a --- /dev/null +++ b/src/backend/storage/smgr/encryption.c @@ -0,0 +1,145 @@ +/*------------------------------------------------------------------------- + * + * encryption.c + * This code handles encryption and decryption of data. + * + * Encryption is done by extension modules loaded by encryption_library GUC. + * The extension module must register itself and provide a cryptography + * implementation. Key setup is left to the extension module. + * + * + * Copyright (c) 2016, PostgreSQL Global Development Group + * + * + * IDENTIFICATION + * src/backend/storage/smgr/encryption.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "catalog/pg_control.h" +#include "storage/bufpage.h" +#include "storage/encryption.h" +#include "miscadmin.h" +#include "fmgr.h" +#include "port.h" + +bool encryption_enabled = false; +bool have_encryption_provider = false; +EncryptionRoutines encryption_hooks; + +/* + * Hook function for encryption providers. The first library to call this + * function gets to provide encryption capability. + */ +void +register_encryption_module(char *name, EncryptionRoutines *enc) +{ + if (!have_encryption_provider) + { + elog(DEBUG1, "Registering encryption module %s", name); + encryption_hooks = *enc; + have_encryption_provider = true; + } +} + +/* + * Encrypts a fixed value into *buf to verify that encryption key is correct. + * Caller provided buf needs to be able to hold at least ENCRYPTION_SAMPLE_SIZE + * bytes. + */ +void +sample_encryption(char *buf) +{ + char tweak[TWEAK_SIZE]; + int i; + for (i = 0; i < TWEAK_SIZE; i++) + tweak[i] = i; + + encrypt_block("postgresqlcrypt", buf, ENCRYPTION_SAMPLE_SIZE, tweak); +} + +/* + * Encrypts one block of data with a specified tweak value. Input and output + * buffer may point to the same location. Size of input must be at least + * ENCRYPTION_BLOCK bytes. Tweak value must be TWEAK_SIZE bytes. + * + * All zero blocks are not encrypted or decrypted to correctly handle relation + * extension. + * + * Must only be called when encryption_enabled is true. + */ +void +encrypt_block(const char *input, char *output, Size size, const char *tweak) +{ + Assert(size >= ENCRYPTION_BLOCK); + Assert(encryption_enabled); + + if (IsAllZero(input, size)) + { + if (input != output) + memset(output, 0, size); + } + else + encryption_hooks.EncryptBlock(input, output, size, tweak); +} + +/* + * Decrypts one block of data with a specified tweak value. Input and output + * buffer may point to the same location. Tweak value must match the one used + * when encrypting. + * + * Must only be called when encryption_enabled is true. + */ +void +decrypt_block(const char *input, char *output, Size size, const char *tweak) +{ + Assert(size >= ENCRYPTION_BLOCK); + Assert(encryption_enabled); + + if (IsAllZero(input, size)) + { + if (input != output) + memset(output, 0, size); + } + else + encryption_hooks.DecryptBlock(input, output, size, tweak); +} + +/* + * Initialize encryption subsystem for use. Must be called before any + * encryptable data is read from or written to data directory. + */ +void +setup_encryption() +{ + char *filename; + + if (encryption_library_string == NULL || encryption_library_string[0] == '\0') + return; + + /* Try to load encryption library */ + filename = pstrdup(encryption_library_string); + + canonicalize_path(filename); + load_file(filename, false); + ereport(DEBUG1, + (errmsg("loaded library \"%s\" for encryption", filename))); + pfree(filename); + + if (have_encryption_provider) + { + encryption_enabled = encryption_hooks.SetupEncryption(); + if (encryption_enabled) + { + if (!IsBootstrapProcessingMode()) + elog(LOG, "data encryption performed by %s", encryption_library_string); + } + else + elog(FATAL, "data encryption could not be initialized"); + } + else + elog(ERROR, "Specified encryption library %s did not provide encryption hooks.", encryption_library_string); +} + diff --git a/src/backend/storage/smgr/md.c b/src/backend/storage/smgr/md.c index f329d15..82b8328 100644 --- a/src/backend/storage/smgr/md.c +++ b/src/backend/storage/smgr/md.c @@ -28,10 +28,12 @@ #include "miscadmin.h" #include "access/xlog.h" #include "catalog/catalog.h" +#include "libpq/md5.h" #include "portability/instr_time.h" #include "postmaster/bgwriter.h" #include "storage/fd.h" #include "storage/bufmgr.h" +#include "storage/encryption.h" #include "storage/relfilenode.h" #include "storage/smgr.h" #include "utils/hsearch.h" @@ -116,7 +118,8 @@ typedef struct _MdfdVec } MdfdVec; static MemoryContext MdCxt; /* context for all MdfdVec objects */ - +static char *md_encryption_buffer; +static char *md_encryption_tweak; /* * In some contexts (currently, standalone backends and the checkpointer) @@ -198,7 +201,9 @@ static MdfdVec *_mdfd_getseg(SMgrRelation reln, ForkNumber forkno, BlockNumber blkno, bool skipFsync, int behavior); static BlockNumber _mdnblocks(SMgrRelation reln, ForkNumber forknum, MdfdVec *seg); - +static void mdtweak(char *tweak, RelFileNode *relnode, ForkNumber forknum, BlockNumber blocknum); +static void mdencrypt(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, char *buffer); +static void mddecrypt(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, char *buffer); /* * mdinit() -- Initialize private state for magnetic disk storage manager. @@ -247,6 +252,9 @@ mdinit(void) HASH_ELEM | HASH_BLOBS | HASH_CONTEXT); pendingUnlinks = NIL; } + + md_encryption_buffer = MemoryContextAllocZero(MdCxt, BLCKSZ); + md_encryption_tweak = MemoryContextAllocZero(MdCxt, TWEAK_SIZE); } /* @@ -542,7 +550,10 @@ mdextend(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, errmsg("could not seek to block %u in file \"%s\": %m", blocknum, FilePathName(v->mdfd_vfd)))); - if ((nbytes = FileWrite(v->mdfd_vfd, buffer, BLCKSZ)) != BLCKSZ) + if (encryption_enabled) + mdencrypt(reln, forknum, blocknum, buffer); + + if ((nbytes = FileWrite(v->mdfd_vfd, encryption_enabled ? md_encryption_buffer : buffer, BLCKSZ)) != BLCKSZ) { if (nbytes < 0) ereport(ERROR, @@ -755,7 +766,7 @@ mdread(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, errmsg("could not seek to block %u in file \"%s\": %m", blocknum, FilePathName(v->mdfd_vfd)))); - nbytes = FileRead(v->mdfd_vfd, buffer, BLCKSZ); + nbytes = FileRead(v->mdfd_vfd, encryption_enabled ? md_encryption_buffer : buffer, BLCKSZ); TRACE_POSTGRESQL_SMGR_MD_READ_DONE(forknum, blocknum, reln->smgr_rnode.node.spcNode, @@ -790,6 +801,8 @@ mdread(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, blocknum, FilePathName(v->mdfd_vfd), nbytes, BLCKSZ))); } + else if (encryption_enabled) + mddecrypt(reln, forknum, blocknum, buffer); } /* @@ -831,7 +844,9 @@ mdwrite(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, errmsg("could not seek to block %u in file \"%s\": %m", blocknum, FilePathName(v->mdfd_vfd)))); - nbytes = FileWrite(v->mdfd_vfd, buffer, BLCKSZ); + if (encryption_enabled) + mdencrypt(reln, forknum, blocknum, buffer); + nbytes = FileWrite(v->mdfd_vfd, encryption_enabled ? md_encryption_buffer : buffer, BLCKSZ); TRACE_POSTGRESQL_SMGR_MD_WRITE_DONE(forknum, blocknum, reln->smgr_rnode.node.spcNode, @@ -1904,3 +1919,54 @@ _mdnblocks(SMgrRelation reln, ForkNumber forknum, MdfdVec *seg) /* note that this calculation will ignore any partial block at EOF */ return (BlockNumber) (len / BLCKSZ); } + +/* + * md files are encrypted block at a time. Tweak will alias higher numbered + * forks for huge tables. + */ +static void +mdtweak(char *tweak, RelFileNode *relnode, ForkNumber forknum, BlockNumber blocknum) +{ + uint32 fork_and_block = (forknum << 24) ^ blocknum; + memcpy(tweak, relnode, 12); + memcpy(tweak+12, &fork_and_block, 4); +} + +static void +mdencrypt(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, char *buffer) +{ + mdtweak(md_encryption_tweak, &(reln->smgr_rnode.node), forknum, blocknum); + encrypt_block(buffer, md_encryption_buffer, BLCKSZ, md_encryption_tweak); +} + +static void +mddecrypt(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, char *dest) +{ + mdtweak(md_encryption_tweak, &(reln->smgr_rnode.node), forknum, blocknum); + decrypt_block(md_encryption_buffer, dest, BLCKSZ, md_encryption_tweak); +} + +/* + * Copying relations between tablespaces/databases means that the tweak values + * of each block will change. This function transcodes a series of blocks with + * new tweak values. Returns the new block number for convenience. + */ +BlockNumber +ReencryptBlock(char *buffer, int blocks, + RelFileNode *srcNode, RelFileNode *dstNode, + ForkNumber srcForkNum, ForkNumber dstForkNum, + BlockNumber blockNum) +{ + char *cur; + char srcTweak[16]; + char dstTweak[16]; + for (cur = buffer; cur < buffer + blocks * BLCKSZ; cur += BLCKSZ) + { + mdtweak(srcTweak, srcNode, srcForkNum, blockNum); + mdtweak(dstTweak, dstNode, dstForkNum, blockNum); + decrypt_block(cur, cur, BLCKSZ, srcTweak); + encrypt_block(cur, cur, BLCKSZ, dstTweak); + blockNum++; + } + return blockNum; +} diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c index b185c1b..4215d5f 100644 --- a/src/backend/tcop/postgres.c +++ b/src/backend/tcop/postgres.c @@ -59,6 +59,7 @@ #include "replication/walsender.h" #include "rewrite/rewriteHandler.h" #include "storage/bufmgr.h" +#include "storage/encryption.h" #include "storage/ipc.h" #include "storage/proc.h" #include "storage/procsignal.h" @@ -3692,6 +3693,8 @@ PostgresMain(int argc, char *argv[], /* Change into DataDir (if under postmaster, was done already) */ ChangeToDataDir(); + setup_encryption(); + /* * Create lockfile for data directory. */ diff --git a/src/backend/utils/init/miscinit.c b/src/backend/utils/init/miscinit.c index d13355b..87d8dcf 100644 --- a/src/backend/utils/init/miscinit.c +++ b/src/backend/utils/init/miscinit.c @@ -1392,6 +1392,7 @@ ValidatePgVersion(const char *path) * GUC variables: lists of library names to be preloaded at postmaster * start and at backend start */ +char *encryption_library_string = NULL; char *session_preload_libraries_string = NULL; char *shared_preload_libraries_string = NULL; char *local_preload_libraries_string = NULL; diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index e246a9c..61c42bb 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -3151,6 +3151,17 @@ static struct config_string ConfigureNamesString[] = }, { + {"encryption_library", PGC_POSTMASTER, CLIENT_CONN_PRELOAD, + gettext_noop("Encryption library to provide transparent data encryption."), + NULL, + GUC_SUPERUSER_ONLY + }, + &encryption_library_string, + "", + NULL, NULL, NULL + }, + + { {"session_preload_libraries", PGC_SUSET, CLIENT_CONN_PRELOAD, gettext_noop("Lists shared libraries to preload into each backend."), NULL, diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index 2d2db7e..924e78a 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -105,6 +105,7 @@ #tcp_keepalives_count = 0 # TCP_KEEPCNT; # 0 selects the system default +#encryption_library = '' #------------------------------------------------------------------------------ # RESOURCE USAGE (except WAL) @@ -173,7 +174,6 @@ #backend_flush_after = 0 # 0 disables, # default is 128kb on linux, 0 otherwise - #------------------------------------------------------------------------------ # WRITE AHEAD LOG #------------------------------------------------------------------------------ diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c index d4a5e7c..318b114 100644 --- a/src/bin/initdb/initdb.c +++ b/src/bin/initdb/initdb.c @@ -142,6 +142,8 @@ static bool do_sync = true; static bool sync_only = false; static bool show_setting = false; static bool data_checksums = false; +static char *data_encryption_module = NULL; +static char *data_encryption_key = NULL; static char *xlog_dir = ""; @@ -279,6 +281,7 @@ static bool check_locale_encoding(const char *locale, int encoding); static void setlocales(void); static void usage(const char *progname); void setup_pgdata(void); +void setup_encryption(void); void setup_bin_paths(const char *argv0); void setup_data_file_paths(void); void setup_locale_encoding(void); @@ -1218,6 +1221,13 @@ setup_config(void) n_buffers * (BLCKSZ / 1024)); conflines = replace_token(conflines, "#shared_buffers = 32MB", repltok); + if (data_encryption_module != NULL) + { + snprintf(repltok, sizeof(repltok), "encryption_library = '%s'", + data_encryption_module); + conflines = replace_token(conflines, "#encryption_library = ''", repltok); + } + #ifdef HAVE_UNIX_SOCKETS snprintf(repltok, sizeof(repltok), "#unix_socket_directories = '%s'", DEFAULT_PGSOCKET_DIR); @@ -2783,6 +2793,20 @@ setup_pgdata(void) putenv(pgdata_set_env); } +void +setup_encryption(void) +{ + char *key = getenv("PGENCRYPTIONKEY"); + if (key != NULL && strlen(key) != 0) + data_encryption_key = pg_strdup(key); + + if (data_encryption_key != NULL) + { + char *pgencryptionkey_set_env; + pgencryptionkey_set_env = psprintf("PGENCRYPTIONKEY=%s", data_encryption_key); + putenv(pgencryptionkey_set_env); + } +} void setup_bin_paths(const char *argv0) @@ -3333,7 +3357,6 @@ initialize_data_directory(void) check_ok(); } - int main(int argc, char *argv[]) { @@ -3364,6 +3387,7 @@ main(int argc, char *argv[]) {"sync-only", no_argument, NULL, 'S'}, {"xlogdir", required_argument, NULL, 'X'}, {"data-checksums", no_argument, NULL, 'k'}, + {"data-encryption", required_argument, NULL, 'K'}, {NULL, 0, NULL, 0} }; @@ -3404,7 +3428,7 @@ main(int argc, char *argv[]) /* process command-line options */ - while ((c = getopt_long(argc, argv, "dD:E:kL:nNU:WA:sST:X:", long_options, &option_index)) != -1) + while ((c = getopt_long(argc, argv, "dD:E:kK:L:nNU:WA:sST:X:", long_options, &option_index)) != -1) { switch (c) { @@ -3456,6 +3480,10 @@ main(int argc, char *argv[]) case 'k': data_checksums = true; break; + case 'K': + if (strlen(optarg) > 0) + data_encryption_module = pg_strdup(optarg); + break; case 'L': share_path = pg_strdup(optarg); break; @@ -3560,6 +3588,7 @@ main(int argc, char *argv[]) setup_bin_paths(argv[0]); + effective_user = get_id(); if (strlen(username) == 0) username = effective_user; @@ -3590,6 +3619,13 @@ main(int argc, char *argv[]) else printf(_("Data page checksums are disabled.\n")); + setup_encryption(); + + if (data_encryption_module != NULL) + printf(_("Using %s for data encryption.\n"), data_encryption_module); + else + printf(_("Data encryption is disabled.\n")); + printf("\n"); initialize_data_directory(); diff --git a/src/bin/pg_controldata/pg_controldata.c b/src/bin/pg_controldata/pg_controldata.c index 96619a2..3188688 100644 --- a/src/bin/pg_controldata/pg_controldata.c +++ b/src/bin/pg_controldata/pg_controldata.c @@ -296,5 +296,14 @@ main(int argc, char *argv[]) (ControlFile->float8ByVal ? _("by value") : _("by reference"))); printf(_("Data page checksum version: %u\n"), ControlFile->data_checksum_version); + printf(_("Data encryption: %s\n"), + ControlFile->data_encrypted ? _("on") : _("off")); + if (ControlFile->data_encrypted) + printf(_("Data encryption fingerprint: %08X%08X%08X%08X\n"), + htonl(((uint32*)ControlFile->encryption_verification)[0]), + htonl(((uint32*)ControlFile->encryption_verification)[1]), + htonl(((uint32*)ControlFile->encryption_verification)[2]), + htonl(((uint32*)ControlFile->encryption_verification)[3]) + ); return 0; } diff --git a/src/bin/pg_upgrade/controldata.c b/src/bin/pg_upgrade/controldata.c index 2601827..6486fc2 100644 --- a/src/bin/pg_upgrade/controldata.c +++ b/src/bin/pg_upgrade/controldata.c @@ -58,6 +58,7 @@ get_control_data(ClusterInfo *cluster, bool live_check) bool got_large_object = false; bool got_date_is_int = false; bool got_data_checksum_version = false; + bool got_data_encrypted = false; char *lc_collate = NULL; char *lc_ctype = NULL; char *lc_monetary = NULL; @@ -129,6 +130,13 @@ get_control_data(ClusterInfo *cluster, bool live_check) got_data_checksum_version = true; } + /* Only in <= 9.6 */ + if (GET_MAJOR_VERSION(cluster->major_version) <= 906) + { + cluster->controldata.data_encrypted = false; + got_data_encrypted = true; + } + /* we have the result of cmd in "output". so parse it line by line now */ while (fgets(bufin, sizeof(bufin), output)) { @@ -412,6 +420,33 @@ get_control_data(ClusterInfo *cluster, bool live_check) cluster->controldata.data_checksum_version = str2uint(p); got_data_checksum_version = true; } + else if ((p = strstr(bufin, "encryption fingerprint")) != NULL) + { + int i; + p = strchr(p, ':'); + + if (p == NULL || strlen(p) <= 1) + pg_fatal("%d: controldata retrieval problem\n", __LINE__); + + cluster->controldata.data_encrypted = true; + + /* Skip the colon and any whitespace after it */ + p = strchr(p, ':'); + if (p == NULL || strlen(p) <= 1) + pg_fatal("%d: controldata retrieval problem\n", __LINE__); + p = strpbrk(p, "01234567890ABCDEF"); + if (p == NULL || strlen(p) <= 1) + pg_fatal("%d: controldata retrieval problem\n", __LINE__); + + /* Make sure it looks like a valid finerprint */ + if (strspn(p, "0123456789ABCDEF") != 32) + pg_fatal("%d: controldata retrieval problem\n", __LINE__); + + for(i = 0; i < 16; i++) + sscanf(p + 2*i, "%2hhx", + cluster->controldata.encryption_verification + i); + got_data_encrypted = true; + } } pclose(output); @@ -466,7 +501,7 @@ get_control_data(ClusterInfo *cluster, bool live_check) !got_index || !got_toast || (!got_large_object && cluster->controldata.ctrl_ver >= LARGE_OBJECT_SIZE_PG_CONTROL_VER) || - !got_date_is_int || !got_data_checksum_version) + !got_date_is_int || !got_data_checksum_version || !got_data_encrypted) { pg_log(PG_REPORT, "The %s cluster lacks some required control information:\n", @@ -529,6 +564,10 @@ get_control_data(ClusterInfo *cluster, bool live_check) if (!got_data_checksum_version) pg_log(PG_REPORT, " data checksum version\n"); + /* value added in Postgres 10 */ + if (!got_data_encrypted) + pg_log(PG_REPORT, " data encryption status\n"); + pg_fatal("Cannot continue without required control information, terminating\n"); } } @@ -593,6 +632,18 @@ check_control_data(ControlData *oldctrl, pg_fatal("old cluster uses data checksums but the new one does not\n"); else if (oldctrl->data_checksum_version != newctrl->data_checksum_version) pg_fatal("old and new cluster pg_controldata checksum versions do not match\n"); + + if (oldctrl->data_encrypted && !newctrl->data_encrypted) + pg_fatal("old cluster is encrypted, but the new one is not\n"); + else if (!oldctrl->data_encrypted && newctrl->data_encrypted) + pg_fatal("old cluster is not encrypted, but the new one is\n"); + else if (oldctrl->data_encrypted && newctrl->data_encrypted) + { + if (oldctrl->encryption_verification != newctrl->encryption_verification) + pg_fatal("encryption keys do not match between old and new cluster\n"); + else + pg_fatal("upgrading encrypted databases is not implemented yet\n"); /* TODO */ + } } diff --git a/src/bin/pg_upgrade/pg_upgrade.h b/src/bin/pg_upgrade/pg_upgrade.h index 10182de..965881f 100644 --- a/src/bin/pg_upgrade/pg_upgrade.h +++ b/src/bin/pg_upgrade/pg_upgrade.h @@ -227,6 +227,8 @@ typedef struct bool date_is_int; bool float8_pass_by_value; bool data_checksum_version; + bool data_encrypted; + uint8 encryption_verification[16]; } ControlData; /* diff --git a/src/include/access/xlogutils.h b/src/include/access/xlogutils.h index d027ea1..5e4d3e2 100644 --- a/src/include/access/xlogutils.h +++ b/src/include/access/xlogutils.h @@ -51,5 +51,7 @@ extern int read_local_xlog_page(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen, XLogRecPtr targetRecPtr, char *cur_page, TimeLineID *pageTLI); +extern void XLogEncryptionTweak(char *tweak, TimeLineID timeline, + XLogSegNo segment, uint32 offset); #endif diff --git a/src/include/catalog/pg_control.h b/src/include/catalog/pg_control.h index 7ba396d..1e21318 100644 --- a/src/include/catalog/pg_control.h +++ b/src/include/catalog/pg_control.h @@ -91,6 +91,11 @@ typedef enum DBState } DBState; /* + * Number of bytes reserved to store encryption sample in ControlFileData. + */ +#define ENCRYPTION_SAMPLE_SIZE 16 + +/* * Contents of pg_control. * * NOTE: try to keep this under 512 bytes so that it will fit on one physical @@ -225,6 +230,11 @@ typedef struct ControlFileData /* Are data pages protected by checksums? Zero if no checksum version */ uint32 data_checksum_version; + /* Is data directory encrypted? */ + bool data_encrypted; + /* Sample value for encryption key verification */ + uint8 encryption_verification[ENCRYPTION_SAMPLE_SIZE]; + /* CRC of all above ... MUST BE LAST! */ pg_crc32c crc; } ControlFileData; diff --git a/src/include/miscadmin.h b/src/include/miscadmin.h index 78545da..d8057f7 100644 --- a/src/include/miscadmin.h +++ b/src/include/miscadmin.h @@ -421,6 +421,7 @@ extern void BaseInit(void); /* in utils/init/miscinit.c */ extern bool IgnoreSystemIndexes; extern PGDLLIMPORT bool process_shared_preload_libraries_in_progress; +extern char *encryption_library_string; extern char *session_preload_libraries_string; extern char *shared_preload_libraries_string; extern char *local_preload_libraries_string; diff --git a/src/include/storage/bufpage.h b/src/include/storage/bufpage.h index 15cebfc..bdceb81 100644 --- a/src/include/storage/bufpage.h +++ b/src/include/storage/bufpage.h @@ -432,4 +432,7 @@ extern void PageIndexDeleteNoCompact(Page page, OffsetNumber *itemnos, extern char *PageSetChecksumCopy(Page page, BlockNumber blkno); extern void PageSetChecksumInplace(Page page, BlockNumber blkno); +/* TODO: Is this the best location for this definition? */ +extern bool IsAllZero(const char *input, Size size); + #endif /* BUFPAGE_H */ diff --git a/src/include/storage/copydir.h b/src/include/storage/copydir.h index 4b7b813..d28df73 100644 --- a/src/include/storage/copydir.h +++ b/src/include/storage/copydir.h @@ -13,7 +13,11 @@ #ifndef COPYDIR_H #define COPYDIR_H -extern void copydir(char *fromdir, char *todir, bool recurse); -extern void copy_file(char *fromfile, char *tofile); +#include "storage/relfilenode.h" + +extern void copydir(char *fromdir, char *todir, RelFileNode *fromNode, RelFileNode *toNode); +extern void copy_file(char *fromfile, char *tofile, RelFileNode *fromNode, RelFileNode *toNode, ForkNumber fromForkNum, ForkNumber toForkNum, int segment); +extern bool parse_filename_for_nontemp_relation(const char *name, + int *oidchars, ForkNumber *fork, int *segment); #endif /* COPYDIR_H */ diff --git a/src/include/storage/encryption.h b/src/include/storage/encryption.h new file mode 100644 index 0000000..bab6412 --- /dev/null +++ b/src/include/storage/encryption.h @@ -0,0 +1,60 @@ +/*------------------------------------------------------------------------- + * + * encryption.h + * Full database encryption support + * + * + * Portions Copyright (c) 1996-2015, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/storage/encryption.h + * + *------------------------------------------------------------------------- + */ +#ifndef ENCRYPTION_H +#define ENCRYPTION_H + +#include "lib/ilist.h" + +#define ENCRYPTION_BLOCK 16 +#define TWEAK_SIZE 16 + +extern PGDLLIMPORT bool encryption_enabled; + + +void setup_encryption(void); +void sample_encryption(char *buf); +void encrypt_block(const char *input, char *output, Size size, + const char *tweak); +void decrypt_block(const char *input, char *output, Size size, + const char *tweak); + +typedef bool (*SetupEncryption_function) (); +typedef void (*EncryptBlock_function) (const char *input, char *output, + Size size, const char *tweak); +typedef void (*DecryptBlock_function) (const char *input, char *output, + Size size, const char *tweak); + +/* + * Hook functions to register an encryption provider. + */ +typedef struct { + dlist_node node; + /* + * Will be called at system initialization time immediately after loading + * the encryption module. Return value indicates if encryption is + * successfully initialized. Returning false will result in a FATAL error. + */ + SetupEncryption_function SetupEncryption; + /* + * Encrypt/decrypt one block of data. Input and output buffers may point + * to the same buffer. Buffer alignment is not guaranteed. Buffer size + * will be at least 16 bytes, but is not guaranteed to be a multiple of 16. + */ + EncryptBlock_function EncryptBlock; + DecryptBlock_function DecryptBlock; +} EncryptionRoutines; + +void register_encryption_module(char *name, EncryptionRoutines *provider); + +#endif /* ENCRYPTION_H */ diff --git a/src/include/storage/smgr.h b/src/include/storage/smgr.h index a8e7877..7e69289 100644 --- a/src/include/storage/smgr.h +++ b/src/include/storage/smgr.h @@ -140,6 +140,12 @@ extern void RememberFsyncRequest(RelFileNode rnode, ForkNumber forknum, extern void ForgetRelationFsyncRequests(RelFileNode rnode, ForkNumber forknum); extern void ForgetDatabaseFsyncRequests(Oid dbid); + +extern BlockNumber ReencryptBlock(char *buffer, int blocks, + RelFileNode *srcNode, RelFileNode *dstNode, + ForkNumber srcForkNum, ForkNumber dstForkNum, + BlockNumber blockNum); + /* smgrtype.c */ extern Datum smgrout(PG_FUNCTION_ARGS); extern Datum smgrin(PG_FUNCTION_ARGS);