Skip to content

slapos
slapos
Commit ce787d53 authored by Alain Takoudjou's avatar Alain Takoudjou
3 Browse files
Options

Update Release Candidate

parents 57575e34 9fe1035e
Show whitespace changes
Inline Side-by-side
Showing with 5403 additions and 780 deletions
component/cmake/buildout.cfg
View file @ ce787d53
[buildout] [buildout]
extends = extends =
../ncurses/buildout.cfg ../ncurses/buildout.cfg
../openssl/buildout.cfg
parts = parts =
cmake cmake
...@@ -11,5 +12,5 @@ shared = true ...@@ -11,5 +12,5 @@ shared = true
url = https://cmake.org/files/v3.7/cmake-3.7.2.tar.gz url = https://cmake.org/files/v3.7/cmake-3.7.2.tar.gz
md5sum = 79bd7e65cd81ea3aa2619484ad6ff25a md5sum = 79bd7e65cd81ea3aa2619484ad6ff25a
environment = environment =
CMAKE_INCLUDE_PATH=${ncurses:location}/include CMAKE_INCLUDE_PATH=${ncurses:location}/include:${openssl:location}/include
CMAKE_LIBRARY_PATH=${ncurses:location}/lib CMAKE_LIBRARY_PATH=${ncurses:location}/lib:${openssl:location}/lib
component/golang/buildout.cfg
View file @ ce787d53
...@@ -68,6 +68,14 @@ md5sum = 6132109d4050da349eadc9f7b0304ef4 ...@@ -68,6 +68,14 @@ md5sum = 6132109d4050da349eadc9f7b0304ef4
environment-extra = environment-extra =
GOROOT_BOOTSTRAP=${golang14:location} GOROOT_BOOTSTRAP=${golang14:location}
[golang1.13]
<= golang-common
url = https://dl.google.com/go/go1.13.9.src.tar.gz
md5sum = 4ad8b04f962be93a32f3021e6f35b3b9
# go1.13 needs go1.4 to bootstrap
environment-extra =
GOROOT_BOOTSTRAP=${golang14:location}
# ---- infrastructure to build Go workspaces / projects ---- # ---- infrastructure to build Go workspaces / projects ----
......
component/haproxy/buildout.cfg
View file @ ce787d53
...@@ -13,8 +13,8 @@ parts = haproxy ...@@ -13,8 +13,8 @@ parts = haproxy
[haproxy] [haproxy]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
shared = true shared = true
url = http://www.haproxy.org/download/1.8/src/haproxy-1.8.23.tar.gz url = http://www.haproxy.org/download/1.8/src/haproxy-1.8.25.tar.gz
md5sum = 6c35b83a9969449c4b79783a2119551e md5sum = ef2164ca3b1ea9011aa271a8cbe030a4
configure-command = true configure-command = true
# If the system is running on Linux 2.6, we use "linux26" as the TARGET, # If the system is running on Linux 2.6, we use "linux26" as the TARGET,
# otherwise use "generic". # otherwise use "generic".
......
component/kumo/buildout.cfg
View file @ ce787d53
[buildout] [buildout]
extends = extends =
../defaults.cfg
../tokyocabinet/buildout.cfg ../tokyocabinet/buildout.cfg
../messagepack/buildout.cfg ../messagepack/buildout.cfg
../openssl/buildout.cfg ../openssl/buildout.cfg
......
component/libmicrohttpd/buildout.cfg 0 → 100644
View file @ ce787d53
[buildout]
extends =
../libiconv/buildout.cfg
../gnutls/buildout.cfg
../curl/buildout.cfg
parts =
libmicrohttpd
[libmicrohttpd]
recipe = slapos.recipe.cmmi
url = https://ftp.gnu.org/gnu/libmicrohttpd/libmicrohttpd-0.9.70.tar.gz
md5sum = dcd6045ecb4ea18c120afedccbd1da74
configure-options =
--with-libcurl=${curl:location}
--with-gnutls=${gnutls:location}
--with-libiconv-prefix=${libiconv:location}
--without-libintl-prefix
component/libtiff/buildout.cfg
View file @ ce787d53
...@@ -19,7 +19,7 @@ configure-options = ...@@ -19,7 +19,7 @@ configure-options =
--disable-lzma --disable-lzma
patch-options = -p1 patch-options = -p1
patches = patches =
${:_profile_base_location_}/tiff_4.0.8-3.debian.patch#9f52aed13d6ae864b3fb46372fe97780 ${:_profile_base_location_}/debian_4.0.8-2+deb9u5.patch#dc9c9cb7f4f9a00100f908e640895185
environment = environment =
CPPFLAGS=-I${libjpeg:location}/include -I${jbigkit:location}/include -I${zlib:location}/include CPPFLAGS=-I${libjpeg:location}/include -I${jbigkit:location}/include -I${zlib:location}/include
LDFLAGS=-L${libjpeg:location}/lib -Wl,-rpath=${libjpeg:location}/lib -L${jbigkit:location}/lib -Wl,-rpath=${jbigkit:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib LDFLAGS=-L${libjpeg:location}/lib -Wl,-rpath=${libjpeg:location}/lib -L${jbigkit:location}/lib -Wl,-rpath=${jbigkit:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
......
component/libtiff/debian_4.0.8-2+deb9u5.patch 0 → 100644
View file @ ce787d53
diff -pur tiff-4.0.4/tools/tiffsplit.c tiff-4.0.4_patch/tools/tiffsplit.c
--- tiff-4.0.4/tools/tiffsplit.c 2015-05-28 15:10:26.000000000 +0200
+++ tiff-4.0.4_patch/tools/tiffsplit.c 2016-02-12 19:15:30.532005041 +0100
@@ -179,8 +179,9 @@ tiffcp(TIFF* in, TIFF* out)
TIFFSetField(out, TIFFTAG_JPEGTABLES, count, table);
}
}
+ uint32 count = 0;
CopyField(TIFFTAG_PHOTOMETRIC, shortv);
- CopyField(TIFFTAG_PREDICTOR, shortv);
+ CopyField2(TIFFTAG_PREDICTOR, count, shortv);
CopyField(TIFFTAG_THRESHHOLDING, shortv);
CopyField(TIFFTAG_FILLORDER, shortv);
CopyField(TIFFTAG_ORIENTATION, shortv);
@@ -188,7 +189,7 @@ tiffcp(TIFF* in, TIFF* out)
CopyField(TIFFTAG_MAXSAMPLEVALUE, shortv);
CopyField(TIFFTAG_XRESOLUTION, floatv);
CopyField(TIFFTAG_YRESOLUTION, floatv);
- CopyField(TIFFTAG_GROUP3OPTIONS, longv);
+ CopyField2(TIFFTAG_GROUP3OPTIONS, count, longv);
CopyField(TIFFTAG_GROUP4OPTIONS, longv);
CopyField(TIFFTAG_RESOLUTIONUNIT, shortv);
CopyField(TIFFTAG_PLANARCONFIG, shortv);
From 02669064e927074819ce1ed39aba0fccaa167717 Mon Sep 17 00:00:00 2001
From: erouault <erouault>
Date: Mon, 29 May 2017 10:12:54 +0000
Subject: [PATCH] * libtiff/tif_color.c: TIFFYCbCrToRGBInit(): stricter
clamping to avoid int32 overflow in TIFFYCbCrtoRGB(). Fixes
https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=1844 Credit to OSS Fuzz
---
ChangeLog | 7 +++++++
libtiff/tif_color.c | 6 +++---
2 files changed, 10 insertions(+), 3 deletions(-)
diff --git a/ChangeLog b/ChangeLog
index ee8d9d08..61116596 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,10 @@
+2017-05-29 Even Rouault <even.rouault at spatialys.com>
+
+ * libtiff/tif_color.c: TIFFYCbCrToRGBInit(): stricter clamping to avoid
+ int32 overflow in TIFFYCbCrtoRGB().
+ Fixes https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=1844
+ Credit to OSS Fuzz
+
2017-05-21 Bob Friesenhahn <bfriesen@simple.dallas.tx.us>
* configure.ac: libtiff 4.0.8 released.
diff --git a/libtiff/tif_color.c b/libtiff/tif_color.c
index 055ed3b2..10a5e66e 100644
--- a/libtiff/tif_color.c
+++ b/libtiff/tif_color.c
@@ -275,10 +275,10 @@ TIFFYCbCrToRGBInit(TIFFYCbCrToRGB* ycbcr, float *luma, float *refBlackWhite)
for (i = 0, x = -128; i < 256; i++, x++) {
int32 Cr = (int32)CLAMPw(Code2V(x, refBlackWhite[4] - 128.0F,
refBlackWhite[5] - 128.0F, 127),
- -128.0F * 64, 128.0F * 64);
+ -128.0F * 32, 128.0F * 32);
int32 Cb = (int32)CLAMPw(Code2V(x, refBlackWhite[2] - 128.0F,
refBlackWhite[3] - 128.0F, 127),
- -128.0F * 64, 128.0F * 64);
+ -128.0F * 32, 128.0F * 32);
ycbcr->Cr_r_tab[i] = (int32)((D1*Cr + ONE_HALF)>>SHIFT);
ycbcr->Cb_b_tab[i] = (int32)((D3*Cb + ONE_HALF)>>SHIFT);
@@ -286,7 +286,7 @@ TIFFYCbCrToRGBInit(TIFFYCbCrToRGB* ycbcr, float *luma, float *refBlackWhite)
ycbcr->Cb_g_tab[i] = D4*Cb + ONE_HALF;
ycbcr->Y_tab[i] =
(int32)CLAMPw(Code2V(x + 128, refBlackWhite[0], refBlackWhite[1], 255),
- -128.0F * 64, 128.0F * 64);
+ -128.0F * 32, 128.0F * 32);
}
}
From 468988860e0dae62ebbf991627c74bcbb4bd256f Mon Sep 17 00:00:00 2001
From: erouault <erouault>
Date: Mon, 29 May 2017 11:29:06 +0000
Subject: [PATCH] * libtiff/tif_getimage.c: initYCbCrConversion(): stricter
validation for refBlackWhite coefficients values. To avoid invalid
float->int32 conversion (when refBlackWhite[0] == 2147483648.f) Fixes
https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=1907 Credit to OSS Fuzz
---
ChangeLog | 8 ++++++++
libtiff/tif_getimage.c | 2 +-
2 files changed, 9 insertions(+), 1 deletion(-)
diff --git a/ChangeLog b/ChangeLog
index a2ddaac2..04881ba7 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,13 @@
2017-05-29 Even Rouault <even.rouault at spatialys.com>
+ * libtiff/tif_getimage.c: initYCbCrConversion(): stricter validation for
+ refBlackWhite coefficients values. To avoid invalid float->int32 conversion
+ (when refBlackWhite[0] == 2147483648.f)
+ Fixes https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=1907
+ Credit to OSS Fuzz
+
+2017-05-29 Even Rouault <even.rouault at spatialys.com>
+
* libtiff/tif_color.c: TIFFYCbCrToRGBInit(): stricter clamping to avoid
int32 overflow in TIFFYCbCrtoRGB().
Fixes https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=1844
diff --git a/libtiff/tif_getimage.c b/libtiff/tif_getimage.c
index dc373abc..a209a7a7 100644
--- a/libtiff/tif_getimage.c
+++ b/libtiff/tif_getimage.c
@@ -2241,7 +2241,7 @@ DECLARESepPutFunc(putseparate8bitYCbCr11tile)
static int isInRefBlackWhiteRange(float f)
{
- return f >= (float)(-0x7FFFFFFF + 128) && f <= (float)0x7FFFFFFF;
+ return f > (float)(-0x7FFFFFFF + 128) && f < (float)0x7FFFFFFF;
}
static int
commit 40448d58fbfad52d2dde5bd18daa30b17fe35fcd
Author: erouault <erouault>
Date: Thu Jun 1 12:44:04 2017 +0000
* libtiff/tif_dirinfo.c, tif_dirread.c: add _TIFFCheckFieldIsValidForCodec(),
and use it in TIFFReadDirectory() so as to ignore fields whose tag is a
codec-specified tag but this codec is not enabled. This avoids TIFFGetField()
to behave differently depending on whether the codec is enabled or not, and
thus can avoid stack based buffer overflows in a number of TIFF utilities
such as tiffsplit, tiffcmp, thumbnail, etc.
Patch derived from 0063-Handle-properly-CODEC-specific-tags.patch
(http://bugzilla.maptools.org/show_bug.cgi?id=2580) by Raphaël Hertzog.
Fixes:
http://bugzilla.maptools.org/show_bug.cgi?id=2580
http://bugzilla.maptools.org/show_bug.cgi?id=2693
http://bugzilla.maptools.org/show_bug.cgi?id=2625 (CVE-2016-10095)
http://bugzilla.maptools.org/show_bug.cgi?id=2564 (CVE-2015-7554)
http://bugzilla.maptools.org/show_bug.cgi?id=2561 (CVE-2016-5318)
http://bugzilla.maptools.org/show_bug.cgi?id=2499 (CVE-2014-8128)
http://bugzilla.maptools.org/show_bug.cgi?id=2441
http://bugzilla.maptools.org/show_bug.cgi?id=2433
diff --git a/ChangeLog b/ChangeLog
index 04881ba7..ebd1a3c0 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,23 @@
+2017-06-01 Even Rouault <even.rouault at spatialys.com>
+
+ * libtiff/tif_dirinfo.c, tif_dirread.c: add _TIFFCheckFieldIsValidForCodec(),
+ and use it in TIFFReadDirectory() so as to ignore fields whose tag is a
+ codec-specified tag but this codec is not enabled. This avoids TIFFGetField()
+ to behave differently depending on whether the codec is enabled or not, and
+ thus can avoid stack based buffer overflows in a number of TIFF utilities
+ such as tiffsplit, tiffcmp, thumbnail, etc.
+ Patch derived from 0063-Handle-properly-CODEC-specific-tags.patch
+ (http://bugzilla.maptools.org/show_bug.cgi?id=2580) by Raphaël Hertzog.
+ Fixes:
+ http://bugzilla.maptools.org/show_bug.cgi?id=2580
+ http://bugzilla.maptools.org/show_bug.cgi?id=2693
+ http://bugzilla.maptools.org/show_bug.cgi?id=2625 (CVE-2016-10095)
+ http://bugzilla.maptools.org/show_bug.cgi?id=2564 (CVE-2015-7554)
+ http://bugzilla.maptools.org/show_bug.cgi?id=2561 (CVE-2016-5318)
+ http://bugzilla.maptools.org/show_bug.cgi?id=2499 (CVE-2014-8128)
+ http://bugzilla.maptools.org/show_bug.cgi?id=2441
+ http://bugzilla.maptools.org/show_bug.cgi?id=2433
+
2017-05-29 Even Rouault <even.rouault at spatialys.com>
* libtiff/tif_getimage.c: initYCbCrConversion(): stricter validation for
diff --git a/libtiff/tif_dir.h b/libtiff/tif_dir.h
index 6af5f3dc..5a380767 100644
--- a/libtiff/tif_dir.h
+++ b/libtiff/tif_dir.h
@@ -1,4 +1,4 @@
-/* $Id: tif_dir.h,v 1.54 2011-02-18 20:53:05 fwarmerdam Exp $ */
+/* $Id: tif_dir.h,v 1.55 2017-06-01 12:44:04 erouault Exp $ */
/*
* Copyright (c) 1988-1997 Sam Leffler
@@ -291,6 +291,7 @@ struct _TIFFField {
extern int _TIFFMergeFields(TIFF*, const TIFFField[], uint32);
extern const TIFFField* _TIFFFindOrRegisterField(TIFF *, uint32, TIFFDataType);
extern TIFFField* _TIFFCreateAnonField(TIFF *, uint32, TIFFDataType);
+extern int _TIFFCheckFieldIsValidForCodec(TIFF *tif, ttag_t tag);
#if defined(__cplusplus)
}
diff --git a/libtiff/tif_dirinfo.c b/libtiff/tif_dirinfo.c
index 23ad0020..4904f540 100644
--- a/libtiff/tif_dirinfo.c
+++ b/libtiff/tif_dirinfo.c
@@ -1,4 +1,4 @@
-/* $Id: tif_dirinfo.c,v 1.126 2016-11-18 02:52:13 bfriesen Exp $ */
+/* $Id: tif_dirinfo.c,v 1.127 2017-06-01 12:44:04 erouault Exp $ */
/*
* Copyright (c) 1988-1997 Sam Leffler
@@ -956,6 +956,109 @@ TIFFMergeFieldInfo(TIFF* tif, const TIFFFieldInfo info[], uint32 n)
return 0;
}
+int
+_TIFFCheckFieldIsValidForCodec(TIFF *tif, ttag_t tag)
+{
+ /* Filter out non-codec specific tags */
+ switch (tag) {
+ /* Shared tags */
+ case TIFFTAG_PREDICTOR:
+ /* JPEG tags */
+ case TIFFTAG_JPEGTABLES:
+ /* OJPEG tags */
+ case TIFFTAG_JPEGIFOFFSET:
+ case TIFFTAG_JPEGIFBYTECOUNT:
+ case TIFFTAG_JPEGQTABLES:
+ case TIFFTAG_JPEGDCTABLES:
+ case TIFFTAG_JPEGACTABLES:
+ case TIFFTAG_JPEGPROC:
+ case TIFFTAG_JPEGRESTARTINTERVAL:
+ /* CCITT* */
+ case TIFFTAG_BADFAXLINES:
+ case TIFFTAG_CLEANFAXDATA:
+ case TIFFTAG_CONSECUTIVEBADFAXLINES:
+ case TIFFTAG_GROUP3OPTIONS:
+ case TIFFTAG_GROUP4OPTIONS:
+ break;
+ default:
+ return 1;
+ }
+ /* Check if codec specific tags are allowed for the current
+ * compression scheme (codec) */
+ switch (tif->tif_dir.td_compression) {
+ case COMPRESSION_LZW:
+ if (tag == TIFFTAG_PREDICTOR)
+ return 1;
+ break;
+ case COMPRESSION_PACKBITS:
+ /* No codec-specific tags */
+ break;
+ case COMPRESSION_THUNDERSCAN:
+ /* No codec-specific tags */
+ break;
+ case COMPRESSION_NEXT:
+ /* No codec-specific tags */
+ break;
+ case COMPRESSION_JPEG:
+ if (tag == TIFFTAG_JPEGTABLES)
+ return 1;
+ break;
+ case COMPRESSION_OJPEG:
+ switch (tag) {
+ case TIFFTAG_JPEGIFOFFSET:
+ case TIFFTAG_JPEGIFBYTECOUNT:
+ case TIFFTAG_JPEGQTABLES:
+ case TIFFTAG_JPEGDCTABLES:
+ case TIFFTAG_JPEGACTABLES:
+ case TIFFTAG_JPEGPROC:
+ case TIFFTAG_JPEGRESTARTINTERVAL:
+ return 1;
+ }
+ break;
+ case COMPRESSION_CCITTRLE:
+ case COMPRESSION_CCITTRLEW:
+ case COMPRESSION_CCITTFAX3:
+ case COMPRESSION_CCITTFAX4:
+ switch (tag) {
+ case TIFFTAG_BADFAXLINES:
+ case TIFFTAG_CLEANFAXDATA:
+ case TIFFTAG_CONSECUTIVEBADFAXLINES:
+ return 1;
+ case TIFFTAG_GROUP3OPTIONS:
+ if (tif->tif_dir.td_compression == COMPRESSION_CCITTFAX3)
+ return 1;
+ break;
+ case TIFFTAG_GROUP4OPTIONS:
+ if (tif->tif_dir.td_compression == COMPRESSION_CCITTFAX4)
+ return 1;
+ break;
+ }
+ break;
+ case COMPRESSION_JBIG:
+ /* No codec-specific tags */
+ break;
+ case COMPRESSION_DEFLATE:
+ case COMPRESSION_ADOBE_DEFLATE:
+ if (tag == TIFFTAG_PREDICTOR)
+ return 1;
+ break;
+ case COMPRESSION_PIXARLOG:
+ if (tag == TIFFTAG_PREDICTOR)
+ return 1;
+ break;
+ case COMPRESSION_SGILOG:
+ case COMPRESSION_SGILOG24:
+ /* No codec-specific tags */
+ break;
+ case COMPRESSION_LZMA:
+ if (tag == TIFFTAG_PREDICTOR)
+ return 1;
+ break;
+
+ }
+ return 0;
+}
+
/* vim: set ts=8 sts=8 sw=8 noet: */
/*
diff --git a/libtiff/tif_dirread.c b/libtiff/tif_dirread.c
index 772ebaf7..acde78b5 100644
--- a/libtiff/tif_dirread.c
+++ b/libtiff/tif_dirread.c
@@ -1,4 +1,4 @@
-/* $Id: tif_dirread.c,v 1.208 2017-04-27 15:46:22 erouault Exp $ */
+/* $Id: tif_dirread.c,v 1.209 2017-06-01 12:44:04 erouault Exp $ */
/*
* Copyright (c) 1988-1997 Sam Leffler
@@ -3580,6 +3580,10 @@ TIFFReadDirectory(TIFF* tif)
goto bad;
dp->tdir_tag=IGNORE;
break;
+ default:
+ if( !_TIFFCheckFieldIsValidForCodec(tif, dp->tdir_tag) )
+ dp->tdir_tag=IGNORE;
+ break;
}
}
}
From fe8d7165956b88df4837034a9161dc5fd20cf67a Mon Sep 17 00:00:00 2001
From: Even Rouault <even.rouault@spatialys.com>
Date: Mon, 26 Jun 2017 15:19:59 +0000
Subject: [PATCH] * libtiff/tif_jbig.c: fix memory leak in error code path of
JBIGDecode() Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2706 Reported
by team OWL337
* libtiff/tif_jpeg.c: error out at decoding time if anticipated libjpeg
---
ChangeLog | 8 +++++++-
libtiff/tif_jbig.c | 1 +
2 files changed, 8 insertions(+), 1 deletion(-)
diff --git a/ChangeLog b/ChangeLog
index bc5096e7..ecd70534 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+2017-06-26 Even Rouault <even.rouault at spatialys.com>
+
+ * libtiff/tif_jbig.c: fix memory leak in error code path of JBIGDecode()
+ Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2706
+ Reported by team OWL337
+
2017-06-01 Even Rouault <even.rouault at spatialys.com>
* libtiff/tif_dirinfo.c, tif_dirread.c: add _TIFFCheckFieldIsValidForCodec(),
diff --git a/libtiff/tif_jbig.c b/libtiff/tif_jbig.c
index 5f5f75e2..c75f31d9 100644
--- a/libtiff/tif_jbig.c
+++ b/libtiff/tif_jbig.c
@@ -94,6 +94,7 @@ static int JBIGDecode(TIFF* tif, uint8* buffer, tmsize_t size, uint16 s)
jbg_strerror(decodeStatus)
#endif
);
+ jbg_dec_free(&decoder);
return 0;
}
From 1077fad562e03d1cad591dd10163dd80ad63ab0e Mon Sep 17 00:00:00 2001
From: Even Rouault <even.rouault@spatialys.com>
Date: Fri, 30 Jun 2017 13:11:18 +0000
Subject: [PATCH] * libtiff/tif_read.c, tiffiop.h: add a
_TIFFReadEncodedStripAndAllocBuffer() function, variant of
TIFFReadEncodedStrip() that allocates the decoded buffer only after a first
successful TIFFFillStrip(). This avoids excessive memory allocation on
corrupted files. * libtiff/tif_getimage.c: use
_TIFFReadEncodedStripAndAllocBuffer(). Fixes
http://bugzilla.maptools.org/show_bug.cgi?id=2708 and
https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=2433 . Credit to OSS
Fuzz
---
ChangeLog | 11 +++++++
libtiff/tif_getimage.c | 59 ++++++++++++++++++++++----------------
libtiff/tif_read.c | 78 +++++++++++++++++++++++++++++++++++++++++++-------
libtiff/tiffiop.h | 5 ++++
4 files changed, 118 insertions(+), 35 deletions(-)
diff --git a/ChangeLog b/ChangeLog
index c969f9e2..6f085e09 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,14 @@
+2017-06-30 Even Rouault <even.rouault at spatialys.com>
+
+ * libtiff/tif_read.c, tiffiop.h: add a _TIFFReadEncodedStripAndAllocBuffer()
+ function, variant of TIFFReadEncodedStrip() that allocates the
+ decoded buffer only after a first successful TIFFFillStrip(). This avoids
+ excessive memory allocation on corrupted files.
+ * libtiff/tif_getimage.c: use _TIFFReadEncodedStripAndAllocBuffer().
+ Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2708 and
+ https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=2433 .
+ Credit to OSS Fuzz
+
2017-06-26 Even Rouault <even.rouault at spatialys.com>
* libtiff/tif_jbig.c: fix memory leak in error code path of JBIGDecode()
diff --git a/libtiff/tif_getimage.c b/libtiff/tif_getimage.c
index cee8e930..cc6e8f30 100644
--- a/libtiff/tif_getimage.c
+++ b/libtiff/tif_getimage.c
@@ -905,26 +905,22 @@ gtStripContig(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
tileContigRoutine put = img->put.contig;
uint32 row, y, nrow, nrowsub, rowstoread;
tmsize_t pos;
- unsigned char* buf;
+ unsigned char* buf = NULL;
uint32 rowsperstrip;
uint16 subsamplinghor,subsamplingver;
uint32 imagewidth = img->width;
tmsize_t scanline;
int32 fromskew, toskew;
int ret = 1, flip;
+ tmsize_t maxstripsize;
TIFFGetFieldDefaulted(tif, TIFFTAG_YCBCRSUBSAMPLING, &subsamplinghor, &subsamplingver);
if( subsamplingver == 0 ) {
TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif), "Invalid vertical YCbCr subsampling");
return (0);
}
-
- buf = (unsigned char*) _TIFFmalloc(TIFFStripSize(tif));
- if (buf == 0) {
- TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif), "No space for strip buffer");
- return (0);
- }
- _TIFFmemset(buf, 0, TIFFStripSize(tif));
+
+ maxstripsize = TIFFStripSize(tif);
flip = setorientation(img);
if (flip & FLIP_VERTICALLY) {
@@ -946,11 +942,12 @@ gtStripContig(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
nrowsub = nrow;
if ((nrowsub%subsamplingver)!=0)
nrowsub+=subsamplingver-nrowsub%subsamplingver;
- if (TIFFReadEncodedStrip(tif,
+ if (_TIFFReadEncodedStripAndAllocBuffer(tif,
TIFFComputeStrip(tif,row+img->row_offset, 0),
- buf,
+ (void**)(&buf),
+ maxstripsize,
((row + img->row_offset)%rowsperstrip + nrowsub) * scanline)==(tmsize_t)(-1)
- && img->stoponerr)
+ && (buf == NULL || img->stoponerr))
{
ret = 0;
break;
@@ -994,8 +991,8 @@ gtStripSeparate(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
{
TIFF* tif = img->tif;
tileSeparateRoutine put = img->put.separate;
- unsigned char *buf;
- unsigned char *p0, *p1, *p2, *pa;
+ unsigned char *buf = NULL;
+ unsigned char *p0 = NULL, *p1 = NULL, *p2 = NULL, *pa = NULL;
uint32 row, y, nrow, rowstoread;
tmsize_t pos;
tmsize_t scanline;
@@ -1014,15 +1011,6 @@ gtStripSeparate(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif), "Integer overflow in %s", "gtStripSeparate");
return (0);
}
- p0 = buf = (unsigned char *)_TIFFmalloc(bufsize);
- if (buf == 0) {
- TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif), "No space for tile buffer");
- return (0);
- }
- _TIFFmemset(buf, 0, bufsize);
- p1 = p0 + stripsize;
- p2 = p1 + stripsize;
- pa = (alpha?(p2+stripsize):NULL);
flip = setorientation(img);
if (flip & FLIP_VERTICALLY) {
@@ -1040,7 +1028,6 @@ gtStripSeparate(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
case PHOTOMETRIC_MINISBLACK:
case PHOTOMETRIC_PALETTE:
colorchannels = 1;
- p2 = p1 = p0;
break;
default:
@@ -1056,7 +1043,31 @@ gtStripSeparate(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
rowstoread = rowsperstrip - (row + img->row_offset) % rowsperstrip;
nrow = (row + rowstoread > h ? h - row : rowstoread);
offset_row = row + img->row_offset;
- if (TIFFReadEncodedStrip(tif, TIFFComputeStrip(tif, offset_row, 0),
+ if( buf == NULL )
+ {
+ if (_TIFFReadEncodedStripAndAllocBuffer(
+ tif, TIFFComputeStrip(tif, offset_row, 0),
+ (void**) &buf, bufsize,
+ ((row + img->row_offset)%rowsperstrip + nrow) * scanline)==(tmsize_t)(-1)
+ && (buf == NULL || img->stoponerr))
+ {
+ ret = 0;
+ break;
+ }
+ p0 = buf;
+ if( colorchannels == 1 )
+ {
+ p2 = p1 = p0;
+ pa = (alpha?(p0+3*stripsize):NULL);
+ }
+ else
+ {
+ p1 = p0 + stripsize;
+ p2 = p1 + stripsize;
+ pa = (alpha?(p2+stripsize):NULL);
+ }
+ }
+ else if (TIFFReadEncodedStrip(tif, TIFFComputeStrip(tif, offset_row, 0),
p0, ((row + img->row_offset)%rowsperstrip + nrow) * scanline)==(tmsize_t)(-1)
&& img->stoponerr)
{
diff --git a/libtiff/tif_read.c b/libtiff/tif_read.c
index fc0072e7..047305ab 100644
--- a/libtiff/tif_read.c
+++ b/libtiff/tif_read.c
@@ -442,18 +442,17 @@ TIFFReadScanline(TIFF* tif, void* buf, uint32 row, uint16 sample)
}
/*
- * Read a strip of data and decompress the specified
- * amount into the user-supplied buffer.
+ * Calculate the strip size according to the number of
+ * rows in the strip (check for truncated last strip on any
+ * of the separations).
*/
-tmsize_t
-TIFFReadEncodedStrip(TIFF* tif, uint32 strip, void* buf, tmsize_t size)
+static tmsize_t TIFFReadEncodedStripGetStripSize(TIFF* tif, uint32 strip, uint16* pplane)
{
static const char module[] = "TIFFReadEncodedStrip";
TIFFDirectory *td = &tif->tif_dir;
uint32 rowsperstrip;
uint32 stripsperplane;
uint32 stripinplane;
- uint16 plane;
uint32 rows;
tmsize_t stripsize;
if (!TIFFCheckRead(tif,0))
@@ -465,23 +464,37 @@ TIFFReadEncodedStrip(TIFF* tif, uint32 strip, void* buf, tmsize_t size)
(unsigned long)td->td_nstrips);
return((tmsize_t)(-1));
}
- /*
- * Calculate the strip size according to the number of
- * rows in the strip (check for truncated last strip on any
- * of the separations).
- */
+
rowsperstrip=td->td_rowsperstrip;
if (rowsperstrip>td->td_imagelength)
rowsperstrip=td->td_imagelength;
stripsperplane= TIFFhowmany_32_maxuint_compat(td->td_imagelength, rowsperstrip);
stripinplane=(strip%stripsperplane);
- plane=(uint16)(strip/stripsperplane);
+ if( pplane ) *pplane=(uint16)(strip/stripsperplane);
rows=td->td_imagelength-stripinplane*rowsperstrip;
if (rows>rowsperstrip)
rows=rowsperstrip;
stripsize=TIFFVStripSize(tif,rows);
if (stripsize==0)
return((tmsize_t)(-1));
+ return stripsize;
+}
+
+/*
+ * Read a strip of data and decompress the specified
+ * amount into the user-supplied buffer.
+ */
+tmsize_t
+TIFFReadEncodedStrip(TIFF* tif, uint32 strip, void* buf, tmsize_t size)
+{
+ static const char module[] = "TIFFReadEncodedStrip";
+ TIFFDirectory *td = &tif->tif_dir;
+ tmsize_t stripsize;
+ uint16 plane;
+
+ stripsize=TIFFReadEncodedStripGetStripSize(tif, strip, &plane);
+ if (stripsize==((tmsize_t)(-1)))
+ return((tmsize_t)(-1));
/* shortcut to avoid an extra memcpy() */
if( td->td_compression == COMPRESSION_NONE &&
@@ -510,6 +523,49 @@ TIFFReadEncodedStrip(TIFF* tif, uint32 strip, void* buf, tmsize_t size)
return(stripsize);
}
+/* Variant of TIFFReadEncodedStrip() that does
+ * * if *buf == NULL, *buf = _TIFFmalloc(bufsizetoalloc) only after TIFFFillStrip() has
+ * suceeded. This avoid excessive memory allocation in case of truncated
+ * file.
+ * * calls regular TIFFReadEncodedStrip() if *buf != NULL
+ */
+tmsize_t
+_TIFFReadEncodedStripAndAllocBuffer(TIFF* tif, uint32 strip,
+ void **buf, tmsize_t bufsizetoalloc,
+ tmsize_t size_to_read)
+{
+ tmsize_t this_stripsize;
+ uint16 plane;
+
+ if( *buf != NULL )
+ {
+ return TIFFReadEncodedStrip(tif, strip, *buf, size_to_read);
+ }
+
+ this_stripsize=TIFFReadEncodedStripGetStripSize(tif, strip, &plane);
+ if (this_stripsize==((tmsize_t)(-1)))
+ return((tmsize_t)(-1));
+
+ if ((size_to_read!=(tmsize_t)(-1))&&(size_to_read<this_stripsize))
+ this_stripsize=size_to_read;
+ if (!TIFFFillStrip(tif,strip))
+ return((tmsize_t)(-1));
+
+ *buf = _TIFFmalloc(bufsizetoalloc);
+ if (*buf == NULL) {
+ TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif), "No space for strip buffer");
+ return((tmsize_t)(-1));
+ }
+ _TIFFmemset(*buf, 0, bufsizetoalloc);
+
+ if ((*tif->tif_decodestrip)(tif,*buf,this_stripsize,plane)<=0)
+ return((tmsize_t)(-1));
+ (*tif->tif_postdecode)(tif,*buf,this_stripsize);
+ return(this_stripsize);
+
+
+}
+
static tmsize_t
TIFFReadRawStrip1(TIFF* tif, uint32 strip, void* buf, tmsize_t size,
const char* module)
diff --git a/libtiff/tiffiop.h b/libtiff/tiffiop.h
index 846ade03..7f0b90f7 100644
--- a/libtiff/tiffiop.h
+++ b/libtiff/tiffiop.h
@@ -365,6 +365,11 @@ extern void* _TIFFCheckRealloc(TIFF*, void*, tmsize_t, tmsize_t, const char*);
extern double _TIFFUInt64ToDouble(uint64);
extern float _TIFFUInt64ToFloat(uint64);
+extern tmsize_t
+_TIFFReadEncodedStripAndAllocBuffer(TIFF* tif, uint32 strip,
+ void **buf, tmsize_t bufsizetoalloc,
+ tmsize_t size_to_read);
+
extern int TIFFInitDumpMode(TIFF*, int);
#ifdef PACKBITS_SUPPORT
extern int TIFFInitPackBits(TIFF*, int);
From 6173a57d39e04d68b139f8c1aa499a24dbe74ba1 Mon Sep 17 00:00:00 2001
From: Even Rouault <even.rouault@spatialys.com>
Date: Fri, 30 Jun 2017 17:29:44 +0000
Subject: [PATCH] * libtiff/tif_dirwrite.c: in
TIFFWriteDirectoryTagCheckedXXXX() functions associated with LONG8/SLONG8
data type, replace assertion that the file is BigTIFF, by a non-fatal error.
Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2712 Reported by team
OWL337
---
ChangeLog | 8 ++++++++
libtiff/tif_dirwrite.c | 20 ++++++++++++++++----
2 files changed, 24 insertions(+), 4 deletions(-)
diff --git a/ChangeLog b/ChangeLog
index 6f085e09..77a64385 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,13 @@
2017-06-30 Even Rouault <even.rouault at spatialys.com>
+ * libtiff/tif_dirwrite.c: in TIFFWriteDirectoryTagCheckedXXXX()
+ functions associated with LONG8/SLONG8 data type, replace assertion that
+ the file is BigTIFF, by a non-fatal error.
+ Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2712
+ Reported by team OWL337
+
+2017-06-30 Even Rouault <even.rouault at spatialys.com>
+
* libtiff/tif_read.c, tiffiop.h: add a _TIFFReadEncodedStripAndAllocBuffer()
function, variant of TIFFReadEncodedStrip() that allocates the
decoded buffer only after a first successful TIFFFillStrip(). This avoids
diff --git a/libtiff/tif_dirwrite.c b/libtiff/tif_dirwrite.c
index 2967da58..8d6686ba 100644
--- a/libtiff/tif_dirwrite.c
+++ b/libtiff/tif_dirwrite.c
@@ -2111,7 +2111,10 @@ TIFFWriteDirectoryTagCheckedLong8(TIFF* tif, uint32* ndir, TIFFDirEntry* dir, ui
{
uint64 m;
assert(sizeof(uint64)==8);
- assert(tif->tif_flags&TIFF_BIGTIFF);
+ if( !(tif->tif_flags&TIFF_BIGTIFF) ) {
+ TIFFErrorExt(tif->tif_clientdata,"TIFFWriteDirectoryTagCheckedLong8","LONG8 not allowed for ClassicTIFF");
+ return(0);
+ }
m=value;
if (tif->tif_flags&TIFF_SWAB)
TIFFSwabLong8(&m);
@@ -2124,7 +2127,10 @@ TIFFWriteDirectoryTagCheckedLong8Array(TIFF* tif, uint32* ndir, TIFFDirEntry* di
{
assert(count<0x20000000);
assert(sizeof(uint64)==8);
- assert(tif->tif_flags&TIFF_BIGTIFF);
+ if( !(tif->tif_flags&TIFF_BIGTIFF) ) {
+ TIFFErrorExt(tif->tif_clientdata,"TIFFWriteDirectoryTagCheckedLong8","LONG8 not allowed for ClassicTIFF");
+ return(0);
+ }
if (tif->tif_flags&TIFF_SWAB)
TIFFSwabArrayOfLong8(value,count);
return(TIFFWriteDirectoryTagData(tif,ndir,dir,tag,TIFF_LONG8,count,count*8,value));
@@ -2136,7 +2142,10 @@ TIFFWriteDirectoryTagCheckedSlong8(TIFF* tif, uint32* ndir, TIFFDirEntry* dir, u
{
int64 m;
assert(sizeof(int64)==8);
- assert(tif->tif_flags&TIFF_BIGTIFF);
+ if( !(tif->tif_flags&TIFF_BIGTIFF) ) {
+ TIFFErrorExt(tif->tif_clientdata,"TIFFWriteDirectoryTagCheckedLong8","SLONG8 not allowed for ClassicTIFF");
+ return(0);
+ }
m=value;
if (tif->tif_flags&TIFF_SWAB)
TIFFSwabLong8((uint64*)(&m));
@@ -2149,7 +2158,10 @@ TIFFWriteDirectoryTagCheckedSlong8Array(TIFF* tif, uint32* ndir, TIFFDirEntry* d
{
assert(count<0x20000000);
assert(sizeof(int64)==8);
- assert(tif->tif_flags&TIFF_BIGTIFF);
+ if( !(tif->tif_flags&TIFF_BIGTIFF) ) {
+ TIFFErrorExt(tif->tif_clientdata,"TIFFWriteDirectoryTagCheckedLong8","SLONG8 not allowed for ClassicTIFF");
+ return(0);
+ }
if (tif->tif_flags&TIFF_SWAB)
TIFFSwabArrayOfLong8((uint64*)value,count);
return(TIFFWriteDirectoryTagData(tif,ndir,dir,tag,TIFF_SLONG8,count,count*8,value));
From 69bfeec247899776b1b396651adb47436e5f1556 Mon Sep 17 00:00:00 2001
From: Even Rouault <even.rouault@spatialys.com>
Date: Sat, 15 Jul 2017 11:13:46 +0000
Subject: [PATCH] * tools/tiff2pdf.c: prevent heap buffer overflow write in
"Raw" mode on PlanarConfig=Contig input images. Fixes
http://bugzilla.maptools.org/show_bug.cgi?id=2715 Reported by team OWL337
---
ChangeLog | 7 +++++++
tools/tiff2pdf.c | 7 ++++++-
2 files changed, 13 insertions(+), 1 deletion(-)
diff --git a/ChangeLog b/ChangeLog
index b4771234..1b5490f3 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,10 @@
+2017-07-15 Even Rouault <even.rouault at spatialys.com>
+
+ * tools/tiff2pdf.c: prevent heap buffer overflow write in "Raw"
+ mode on PlanarConfig=Contig input images.
+ Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2715
+ Reported by team OWL337
+
2017-06-30 Even Rouault <even.rouault at spatialys.com>
* libtiff/tif_dirwrite.c: in TIFFWriteDirectoryTagCheckedXXXX()
diff --git a/tools/tiff2pdf.c b/tools/tiff2pdf.c
index db196e04..cd1e2358 100644
--- a/tools/tiff2pdf.c
+++ b/tools/tiff2pdf.c
@@ -1737,7 +1737,12 @@ void t2p_read_tiff_data(T2P* t2p, TIFF* input){
return;
t2p->pdf_transcode = T2P_TRANSCODE_ENCODE;
- if(t2p->pdf_nopassthrough==0){
+ /* It seems that T2P_TRANSCODE_RAW mode doesn't support separate->contig */
+ /* conversion. At least t2p_read_tiff_size and t2p_read_tiff_size_tile */
+ /* do not take into account the number of samples, and thus */
+ /* that can cause heap buffer overflows such as in */
+ /* http://bugzilla.maptools.org/show_bug.cgi?id=2715 */
+ if(t2p->pdf_nopassthrough==0 && t2p->tiff_planar!=PLANARCONFIG_SEPARATE){
#ifdef CCITT_SUPPORT
if(t2p->tiff_compression==COMPRESSION_CCITTFAX4
){
From dc02f9050311a90b3c0655147cee09bfa7081cfc Mon Sep 17 00:00:00 2001
From: Even Rouault <even.rouault@spatialys.com>
Date: Sat, 15 Jul 2017 13:23:09 +0000
Subject: [PATCH] * libtiff/tif_read.c: add protection against excessive memory
allocation attempts in TIFFReadDirEntryArray() on short files. Effective for
mmap'ed case. And non-mmap'ed case, but restricted to 64bit builds. Fixes
http://bugzilla.maptools.org/show_bug.cgi?id=2675
---
ChangeLog | 8 +++++
libtiff/tif_dirread.c | 89 ++++++++++++++++++++++++++++++++++++++++++++++++---
2 files changed, 92 insertions(+), 5 deletions(-)
diff --git a/ChangeLog b/ChangeLog
index 932ddee5..e8a2be5b 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,13 @@
2017-07-15 Even Rouault <even.rouault at spatialys.com>
+ * libtiff/tif_read.c: add protection against excessive memory
+ allocation attempts in TIFFReadDirEntryArray() on short files.
+ Effective for mmap'ed case. And non-mmap'ed case, but restricted
+ to 64bit builds.
+ Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2675
+
+2017-07-15 Even Rouault <even.rouault at spatialys.com>
+
* tools/tiff2pdf.c: prevent heap buffer overflow write in "Raw"
mode on PlanarConfig=Contig input images.
Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2715
diff --git a/libtiff/tif_dirread.c b/libtiff/tif_dirread.c
index 99ca5200..411d1a45 100644
--- a/libtiff/tif_dirread.c
+++ b/libtiff/tif_dirread.c
@@ -765,6 +765,65 @@ static enum TIFFReadDirEntryErr TIFFRead
}
}
+#define INITIAL_THRESHOLD (1024 * 1024)
+#define THRESHOLD_MULTIPLIER 10
+#define MAX_THRESHOLD (THRESHOLD_MULTIPLIER * THRESHOLD_MULTIPLIER * THRESHOLD_MULTIPLIER * INITIAL_THRESHOLD)
+
+static enum TIFFReadDirEntryErr TIFFReadDirEntryDataAndRealloc(
+ TIFF* tif, uint64 offset, tmsize_t size, void** pdest)
+{
+#if SIZEOF_VOIDP == 8 || SIZEOF_SIZE_T == 8
+ tmsize_t threshold = INITIAL_THRESHOLD;
+#endif
+ tmsize_t already_read = 0;
+
+ assert( !isMapped(tif) );
+
+ if (!SeekOK(tif,offset))
+ return(TIFFReadDirEntryErrIo);
+
+ /* On 64 bit processes, read first a maximum of 1 MB, then 10 MB, etc */
+ /* so as to avoid allocating too much memory in case the file is too */
+ /* short. We could ask for the file size, but this might be */
+ /* expensive with some I/O layers (think of reading a gzipped file) */
+ /* Restrict to 64 bit processes, so as to avoid reallocs() */
+ /* on 32 bit processes where virtual memory is scarce. */
+ while( already_read < size )
+ {
+ void* new_dest;
+ tmsize_t bytes_read;
+ tmsize_t to_read = size - already_read;
+#if SIZEOF_VOIDP == 8 || SIZEOF_SIZE_T == 8
+ if( to_read >= threshold && threshold < MAX_THRESHOLD )
+ {
+ to_read = threshold;
+ threshold *= THRESHOLD_MULTIPLIER;
+ }
+#endif
+
+ new_dest = (uint8*) _TIFFrealloc(
+ *pdest, already_read + to_read);
+ if( new_dest == NULL )
+ {
+ TIFFErrorExt(tif->tif_clientdata, tif->tif_name,
+ "Failed to allocate memory for %s "
+ "(%ld elements of %ld bytes each)",
+ "TIFFReadDirEntryArray",
+ (long) 1, (long) already_read + to_read);
+ return TIFFReadDirEntryErrAlloc;
+ }
+ *pdest = new_dest;
+
+ bytes_read = TIFFReadFile(tif,
+ (char*)*pdest + already_read, to_read);
+ already_read += bytes_read;
+ if (bytes_read != to_read) {
+ return TIFFReadDirEntryErrIo;
+ }
+ }
+ return TIFFReadDirEntryErrOk;
+}
+
static enum TIFFReadDirEntryErr TIFFReadDirEntryArray(TIFF* tif, TIFFDirEntry* direntry, uint32* count, uint32 desttypesize, void** value)
{
int typesize;
@@ -791,9 +851,22 @@ static enum TIFFReadDirEntryErr TIFFReadDirEntryArrayWithLimit(
*count=(uint32)direntry->tdir_count;
datasize=(*count)*typesize;
assert((tmsize_t)datasize>0);
- data=_TIFFCheckMalloc(tif, *count, typesize, "ReadDirEntryArray");
- if (data==0)
- return(TIFFReadDirEntryErrAlloc);
+
+ if( isMapped(tif) && datasize > tif->tif_size )
+ return TIFFReadDirEntryErrIo;
+
+ if( !isMapped(tif) &&
+ (((tif->tif_flags&TIFF_BIGTIFF) && datasize > 8) ||
+ (!(tif->tif_flags&TIFF_BIGTIFF) && datasize > 4)) )
+ {
+ data = NULL;
+ }
+ else
+ {
+ data=_TIFFCheckMalloc(tif, *count, typesize, "ReadDirEntryArray");
+ if (data==0)
+ return(TIFFReadDirEntryErrAlloc);
+ }
if (!(tif->tif_flags&TIFF_BIGTIFF))
{
if (datasize<=4)
@@ -804,7 +877,10 @@ static enum TIFFReadDirEntryErr TIFFReadDirEntryArrayWithLimit(
uint32 offset = direntry->tdir_offset.toff_long;
if (tif->tif_flags&TIFF_SWAB)
TIFFSwabLong(&offset);
- err=TIFFReadDirEntryData(tif,(uint64)offset,(tmsize_t)datasize,data);
+ if( isMapped(tif) )
+ err=TIFFReadDirEntryData(tif,(uint64)offset,(tmsize_t)datasize,data);
+ else
+ err=TIFFReadDirEntryDataAndRealloc(tif,(uint64)offset,(tmsize_t)datasize,&data);
if (err!=TIFFReadDirEntryErrOk)
{
_TIFFfree(data);
@@ -822,7 +898,10 @@ static enum TIFFReadDirEntryErr TIFFReadDirEntryArrayWithLimit(
uint64 offset = direntry->tdir_offset.toff_long8;
if (tif->tif_flags&TIFF_SWAB)
TIFFSwabLong8(&offset);
- err=TIFFReadDirEntryData(tif,offset,(tmsize_t)datasize,data);
+ if( isMapped(tif) )
+ err=TIFFReadDirEntryData(tif,(uint64)offset,(tmsize_t)datasize,data);
+ else
+ err=TIFFReadDirEntryDataAndRealloc(tif,(uint64)offset,(tmsize_t)datasize,&data);
if (err!=TIFFReadDirEntryErrOk)
{
_TIFFfree(data);
From f91ca83a21a6a583050e5a5755ce1441b2bf1d7e Mon Sep 17 00:00:00 2001
From: Even Rouault <even.rouault@spatialys.com>
Date: Wed, 23 Aug 2017 13:21:41 +0000
Subject: [PATCH] * libtiff/tif_dirwrite.c: replace assertion related to not
finding the SubIFD tag by runtime check. Fixes
http://bugzilla.maptools.org/show_bug.cgi?id=2727 Reported by team OWL337
---
ChangeLog | 7 +++++++
libtiff/tif_dirwrite.c | 7 ++++++-
2 files changed, 13 insertions(+), 1 deletion(-)
diff --git a/ChangeLog b/ChangeLog
index 3da2b704..87554768 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,10 @@
+2017-08-23 Even Rouault <even.rouault at spatialys.com>
+
+ * libtiff/tif_dirwrite.c: replace assertion related to not finding the
+ SubIFD tag by runtime check.
+ Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2727
+ Reported by team OWL337
+
2017-07-15 Even Rouault <even.rouault at spatialys.com>
* libtiff/tif_read.c: add protection against excessive memory
diff --git a/libtiff/tif_dirwrite.c b/libtiff/tif_dirwrite.c
index 38edb3fb..a85f0627 100644
--- a/libtiff/tif_dirwrite.c
+++ b/libtiff/tif_dirwrite.c
@@ -821,7 +821,12 @@ TIFFWriteDirectorySec(TIFF* tif, int isimage, int imagedone, uint64* pdiroff)
TIFFDirEntry* nb;
for (na=0, nb=dir; ; na++, nb++)
{
- assert(na<ndir);
+ if( na == ndir )
+ {
+ TIFFErrorExt(tif->tif_clientdata,module,
+ "Cannot find SubIFD tag");
+ goto bad;
+ }
if (nb->tdir_tag==TIFFTAG_SUBIFD)
break;
}
From b6af137bf9ef852f1a48a50a5afb88f9e9da01cc Mon Sep 17 00:00:00 2001
From: Even Rouault <even.rouault@spatialys.com>
Date: Wed, 23 Aug 2017 13:33:42 +0000
Subject: [PATCH] * libtiff/tif_dirwrite.c: replace assertion to tag value not
fitting on uint32 when selecting the value of SubIFD tag by runtime check (in
TIFFWriteDirectoryTagSubifd()). Fixes
http://bugzilla.maptools.org/show_bug.cgi?id=2728 Reported by team OWL337
SubIFD tag by runtime check (in TIFFWriteDirectorySec())
---
ChangeLog | 10 +++++++++-
libtiff/tif_dirwrite.c | 9 ++++++++-
2 files changed, 17 insertions(+), 2 deletions(-)
diff --git a/ChangeLog b/ChangeLog
index 87554768..58d5e0cc 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,7 +1,15 @@
+2017-08-23 Even Rouault <even.rouault at spatialys.com>
+
+ * libtiff/tif_dirwrite.c: replace assertion to tag value not fitting
+ on uint32 when selecting the value of SubIFD tag by runtime check
+ (in TIFFWriteDirectoryTagSubifd()).
+ Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2728
+ Reported by team OWL337
+
2017-08-23 Even Rouault <even.rouault at spatialys.com>
* libtiff/tif_dirwrite.c: replace assertion related to not finding the
- SubIFD tag by runtime check.
+ SubIFD tag by runtime check (in TIFFWriteDirectorySec())
Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2727
Reported by team OWL337
diff --git a/libtiff/tif_dirwrite.c b/libtiff/tif_dirwrite.c
index a85f0627..cad0a498 100644
--- a/libtiff/tif_dirwrite.c
+++ b/libtiff/tif_dirwrite.c
@@ -1949,7 +1949,14 @@ TIFFWriteDirectoryTagSubifd(TIFF* tif, uint32* ndir, TIFFDirEntry* dir)
for (p=0; p < tif->tif_dir.td_nsubifd; p++)
{
assert(pa != 0);
- assert(*pa <= 0xFFFFFFFFUL);
+
+ /* Could happen if an classicTIFF has a SubIFD of type LONG8 (which is illegal) */
+ if( *pa > 0xFFFFFFFFUL)
+ {
+ TIFFErrorExt(tif->tif_clientdata,module,"Illegal value for SubIFD tag");
+ _TIFFfree(o);
+ return(0);
+ }
*pb++=(uint32)(*pa++);
}
n=TIFFWriteDirectoryTagCheckedIfdArray(tif,ndir,dir,TIFFTAG_SUBIFD,tif->tif_dir.td_nsubifd,o);
From c6f41df7b581402dfba3c19a1e3df4454c551a01 Mon Sep 17 00:00:00 2001
From: Even Rouault <even.rouault@spatialys.com>
Date: Sun, 31 Dec 2017 15:09:41 +0100
Subject: [PATCH] libtiff/tif_print.c: TIFFPrintDirectory(): fix null pointer dereference on corrupted file. Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2770
---
libtiff/tif_print.c | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/libtiff/tif_print.c b/libtiff/tif_print.c
index 9959d35..8deceb2 100644
--- a/libtiff/tif_print.c
+++ b/libtiff/tif_print.c
@@ -667,13 +667,13 @@ TIFFPrintDirectory(TIFF* tif, FILE* fd, long flags)
#if defined(__WIN32__) && (defined(_MSC_VER) || defined(__MINGW32__))
fprintf(fd, " %3lu: [%8I64u, %8I64u]\n",
(unsigned long) s,
- (unsigned __int64) td->td_stripoffset[s],
- (unsigned __int64) td->td_stripbytecount[s]);
+ td->td_stripoffset ? (unsigned __int64) td->td_stripoffset[s] : 0,
+ td->td_stripbytecount ? (unsigned __int64) td->td_stripbytecount[s] : 0);
#else
fprintf(fd, " %3lu: [%8llu, %8llu]\n",
(unsigned long) s,
- (unsigned long long) td->td_stripoffset[s],
- (unsigned long long) td->td_stripbytecount[s]);
+ td->td_stripoffset ? (unsigned long long) td->td_stripoffset[s] : 0,
+ td->td_stripbytecount ? (unsigned long long) td->td_stripbytecount[s] : 0);
#endif
}
}
--
libgit2 0.26.0
diff --git a/libtiff/tif_dir.c b/libtiff/tif_dir.c
index 2ccaf44..cbf2b69 100644
--- a/libtiff/tif_dir.c
+++ b/libtiff/tif_dir.c
@@ -1065,6 +1065,9 @@ _TIFFVGetField(TIFF* tif, uint32 tag, va_list ap)
if (td->td_samplesperpixel - td->td_extrasamples > 1) {
*va_arg(ap, uint16**) = td->td_transferfunction[1];
*va_arg(ap, uint16**) = td->td_transferfunction[2];
+ } else {
+ *va_arg(ap, uint16**) = NULL;
+ *va_arg(ap, uint16**) = NULL;
}
break;
case TIFFTAG_REFERENCEBLACKWHITE:
diff --git a/tools/tiff2pdf.c b/tools/tiff2pdf.c
index d1a9b09..484776c 100644
--- a/tools/tiff2pdf.c
+++ b/tools/tiff2pdf.c
@@ -237,7 +237,7 @@ typedef struct {
float tiff_whitechromaticities[2];
float tiff_primarychromaticities[6];
float tiff_referenceblackwhite[2];
- float* tiff_transferfunction[3];
+ uint16* tiff_transferfunction[3];
int pdf_image_interpolate; /* 0 (default) : do not interpolate,
1 : interpolate */
uint16 tiff_transferfunctioncount;
@@ -1047,6 +1047,8 @@ void t2p_read_tiff_init(T2P* t2p, TIFF* input){
uint16 pagen=0;
uint16 paged=0;
uint16 xuint16=0;
+ uint16 tiff_transferfunctioncount=0;
+ uint16* tiff_transferfunction[3];
directorycount=TIFFNumberOfDirectories(input);
t2p->tiff_pages = (T2P_PAGE*) _TIFFmalloc(TIFFSafeMultiply(tmsize_t,directorycount,sizeof(T2P_PAGE)));
@@ -1147,26 +1149,48 @@ void t2p_read_tiff_init(T2P* t2p, TIFF* input){
}
#endif
if (TIFFGetField(input, TIFFTAG_TRANSFERFUNCTION,
- &(t2p->tiff_transferfunction[0]),
- &(t2p->tiff_transferfunction[1]),
- &(t2p->tiff_transferfunction[2]))) {
- if((t2p->tiff_transferfunction[1] != (float*) NULL) &&
- (t2p->tiff_transferfunction[2] != (float*) NULL) &&
- (t2p->tiff_transferfunction[1] !=
- t2p->tiff_transferfunction[0])) {
- t2p->tiff_transferfunctioncount = 3;
- t2p->tiff_pages[i].page_extra += 4;
- t2p->pdf_xrefcount += 4;
- } else {
- t2p->tiff_transferfunctioncount = 1;
- t2p->tiff_pages[i].page_extra += 2;
- t2p->pdf_xrefcount += 2;
- }
- if(t2p->pdf_minorversion < 2)
- t2p->pdf_minorversion = 2;
+ &(tiff_transferfunction[0]),
+ &(tiff_transferfunction[1]),
+ &(tiff_transferfunction[2]))) {
+
+ if((tiff_transferfunction[1] != (uint16*) NULL) &&
+ (tiff_transferfunction[2] != (uint16*) NULL)
+ ) {
+ tiff_transferfunctioncount=3;
+ } else {
+ tiff_transferfunctioncount=1;
+ }
} else {
- t2p->tiff_transferfunctioncount=0;
+ tiff_transferfunctioncount=0;
}
+
+ if (i > 0){
+ if (tiff_transferfunctioncount != t2p->tiff_transferfunctioncount){
+ TIFFError(
+ TIFF2PDF_MODULE,
+ "Different transfer function on page %d",
+ i);
+ t2p->t2p_error = T2P_ERR_ERROR;
+ return;
+ }
+ }
+
+ t2p->tiff_transferfunctioncount = tiff_transferfunctioncount;
+ t2p->tiff_transferfunction[0] = tiff_transferfunction[0];
+ t2p->tiff_transferfunction[1] = tiff_transferfunction[1];
+ t2p->tiff_transferfunction[2] = tiff_transferfunction[2];
+ if(tiff_transferfunctioncount == 3){
+ t2p->tiff_pages[i].page_extra += 4;
+ t2p->pdf_xrefcount += 4;
+ if(t2p->pdf_minorversion < 2)
+ t2p->pdf_minorversion = 2;
+ } else if (tiff_transferfunctioncount == 1){
+ t2p->tiff_pages[i].page_extra += 2;
+ t2p->pdf_xrefcount += 2;
+ if(t2p->pdf_minorversion < 2)
+ t2p->pdf_minorversion = 2;
+ }
+
if( TIFFGetField(
input,
TIFFTAG_ICCPROFILE,
@@ -1827,10 +1851,9 @@ void t2p_read_tiff_data(T2P* t2p, TIFF* input){
&(t2p->tiff_transferfunction[0]),
&(t2p->tiff_transferfunction[1]),
&(t2p->tiff_transferfunction[2]))) {
- if((t2p->tiff_transferfunction[1] != (float*) NULL) &&
- (t2p->tiff_transferfunction[2] != (float*) NULL) &&
- (t2p->tiff_transferfunction[1] !=
- t2p->tiff_transferfunction[0])) {
+ if((t2p->tiff_transferfunction[1] != (uint16*) NULL) &&
+ (t2p->tiff_transferfunction[2] != (uint16*) NULL)
+ ) {
t2p->tiff_transferfunctioncount=3;
} else {
t2p->tiff_transferfunctioncount=1;
From 3719385a3fac5cfb20b487619a5f08abbf967cf8 Mon Sep 17 00:00:00 2001
From: Even Rouault <even.rouault@spatialys.com>
Date: Sun, 11 Mar 2018 11:14:01 +0100
Subject: [PATCH] ChopUpSingleUncompressedStrip: avoid memory exhaustion (CVE-2017-11613)
In ChopUpSingleUncompressedStrip(), if the computed number of strips is big
enough and we are in read only mode, validate that the file size is consistent
with that number of strips to avoid useless attempts at allocating a lot of
memory for the td_stripbytecount and td_stripoffset arrays.
Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2724
---
libtiff/tif_dirread.c | 11 +++++++++++
1 file changed, 11 insertions(+)
diff --git a/libtiff/tif_dirread.c b/libtiff/tif_dirread.c
index 3fc0c8e..1a3259c 100644
--- a/libtiff/tif_dirread.c
+++ b/libtiff/tif_dirread.c
@@ -5653,6 +5653,17 @@ ChopUpSingleUncompressedStrip(TIFF* tif)
if( nstrips == 0 )
return;
+ /* If we are going to allocate a lot of memory, make sure that the */
+ /* file is as big as needed */
+ if( tif->tif_mode == O_RDONLY &&
+ nstrips > 1000000 &&
+ (tif->tif_dir.td_stripoffset[0] >= TIFFGetFileSize(tif) ||
+ tif->tif_dir.td_stripbytecount[0] >
+ TIFFGetFileSize(tif) - tif->tif_dir.td_stripoffset[0]) )
+ {
+ return;
+ }
+
newcounts = (uint64*) _TIFFCheckMalloc(tif, nstrips, sizeof (uint64),
"for chopped \"StripByteCounts\" array");
newoffsets = (uint64*) _TIFFCheckMalloc(tif, nstrips, sizeof (uint64),
--
libgit2 0.27.0
From 7a092f8af2568d61993a8cc2e7a35a998d7d37be Mon Sep 17 00:00:00 2001
From: Even Rouault <even.rouault@spatialys.com>
Date: Sat, 17 Mar 2018 09:36:29 +0100
Subject: [PATCH] ChopUpSingleUncompressedStrip: avoid memory exhaustion (CVE-2017-11613)
Rework fix done in 3719385a3fac5cfb20b487619a5f08abbf967cf8 to work in more
cases like https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=6979.
Credit to OSS Fuzz
Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2724
---
libtiff/tif_dirread.c | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git a/libtiff/tif_dirread.c b/libtiff/tif_dirread.c
index 1a3259c..6baa7b3 100644
--- a/libtiff/tif_dirread.c
+++ b/libtiff/tif_dirread.c
@@ -5657,9 +5657,8 @@ ChopUpSingleUncompressedStrip(TIFF* tif)
/* file is as big as needed */
if( tif->tif_mode == O_RDONLY &&
nstrips > 1000000 &&
- (tif->tif_dir.td_stripoffset[0] >= TIFFGetFileSize(tif) ||
- tif->tif_dir.td_stripbytecount[0] >
- TIFFGetFileSize(tif) - tif->tif_dir.td_stripoffset[0]) )
+ (offset >= TIFFGetFileSize(tif) ||
+ stripbytes > (TIFFGetFileSize(tif) - offset) / (nstrips - 1)) )
{
return;
}
--
libgit2 0.27.0
From be4c85b16e8801a16eec25e80eb9f3dd6a96731b Mon Sep 17 00:00:00 2001
From: Hugo Lefeuvre <hle@debian.org>
Date: Sun, 8 Apr 2018 14:07:08 -0400
Subject: [PATCH] Fix NULL pointer dereference in TIFFPrintDirectory
The TIFFPrintDirectory function relies on the following assumptions,
supposed to be guaranteed by the specification:
(a) A Transfer Function field is only present if the TIFF file has
photometric type < 3.
(b) If SamplesPerPixel > Color Channels, then the ExtraSamples field
has count SamplesPerPixel - (Color Channels) and contains
information about supplementary channels.
While respect of (a) and (b) are essential for the well functioning of
TIFFPrintDirectory, no checks are realized neither by the callee nor
by TIFFPrintDirectory itself. Hence, following scenarios might happen
and trigger the NULL pointer dereference:
(1) TIFF File of photometric type 4 or more has illegal Transfer
Function field.
(2) TIFF File has photometric type 3 or less and defines a
SamplesPerPixel field such that SamplesPerPixel > Color Channels
without defining all extra samples in the ExtraSamples fields.
In this patch, we address both issues with respect of the following
principles:
(A) In the case of (1), the defined transfer table should be printed
safely even if it isn't 'legal'. This allows us to avoid expensive
checks in TIFFPrintDirectory. Also, it is quite possible that
an alternative photometric type would be developed (not part of the
standard) and would allow definition of Transfer Table. We want
libtiff to be able to handle this scenario out of the box.
(B) In the case of (2), the transfer table should be printed at its
right size, that is if TIFF file has photometric type Palette
then the transfer table should have one row and not three, even
if two extra samples are declared.
In order to fulfill (A) we simply add a new 'i < 3' end condition to
the broken TIFFPrintDirectory loop. This makes sure that in any case
where (b) would be respected but not (a), everything stays fine.
(B) is fulfilled by the loop condition
'i < td->td_samplesperpixel - td->td_extrasamples'. This is enough as
long as (b) is respected.
Naturally, we also make sure (b) is respected. This is done in the
TIFFReadDirectory function by making sure any non-color channel is
counted in ExtraSamples.
This commit addresses CVE-2018-7456.
---
libtiff/tif_dirread.c | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
libtiff/tif_print.c | 2 +-
2 files changed, 63 insertions(+), 1 deletion(-)
diff --git a/libtiff/tif_dirread.c b/libtiff/tif_dirread.c
index 6baa7b3..af5b84a 100644
--- a/libtiff/tif_dirread.c
+++ b/libtiff/tif_dirread.c
@@ -166,6 +166,7 @@ static int TIFFFetchStripThing(TIFF* tif, TIFFDirEntry* dir, uint32 nstrips, uin
static int TIFFFetchSubjectDistance(TIFF*, TIFFDirEntry*);
static void ChopUpSingleUncompressedStrip(TIFF*);
static uint64 TIFFReadUInt64(const uint8 *value);
+static int _TIFFGetMaxColorChannels(uint16 photometric);
static int _TIFFFillStrilesInternal( TIFF *tif, int loadStripByteCount );
@@ -3484,6 +3485,35 @@ static void TIFFReadDirEntryOutputErr(TIFF* tif, enum TIFFReadDirEntryErr err, c
}
/*
+ * Return the maximum number of color channels specified for a given photometric
+ * type. 0 is returned if photometric type isn't supported or no default value
+ * is defined by the specification.
+ */
+static int _TIFFGetMaxColorChannels( uint16 photometric )
+{
+ switch (photometric) {
+ case PHOTOMETRIC_PALETTE:
+ case PHOTOMETRIC_MINISWHITE:
+ case PHOTOMETRIC_MINISBLACK:
+ return 1;
+ case PHOTOMETRIC_YCBCR:
+ case PHOTOMETRIC_RGB:
+ case PHOTOMETRIC_CIELAB:
+ return 3;
+ case PHOTOMETRIC_SEPARATED:
+ case PHOTOMETRIC_MASK:
+ return 4;
+ case PHOTOMETRIC_LOGL:
+ case PHOTOMETRIC_LOGLUV:
+ case PHOTOMETRIC_CFA:
+ case PHOTOMETRIC_ITULAB:
+ case PHOTOMETRIC_ICCLAB:
+ default:
+ return 0;
+ }
+}
+
+/*
* Read the next TIFF directory from a file and convert it to the internal
* format. We read directories sequentially.
*/
@@ -3499,6 +3529,7 @@ TIFFReadDirectory(TIFF* tif)
uint32 fii=FAILED_FII;
toff_t nextdiroff;
int bitspersample_read = FALSE;
+ int color_channels;
tif->tif_diroff=tif->tif_nextdiroff;
if (!TIFFCheckDirOffset(tif,tif->tif_nextdiroff))
@@ -4003,6 +4034,37 @@ TIFFReadDirectory(TIFF* tif)
}
}
}
+
+ /*
+ * Make sure all non-color channels are extrasamples.
+ * If it's not the case, define them as such.
+ */
+ color_channels = _TIFFGetMaxColorChannels(tif->tif_dir.td_photometric);
+ if (color_channels && tif->tif_dir.td_samplesperpixel - tif->tif_dir.td_extrasamples > color_channels) {
+ uint16 old_extrasamples;
+ uint16 *new_sampleinfo;
+
+ TIFFWarningExt(tif->tif_clientdata,module, "Sum of Photometric type-related "
+ "color channels and ExtraSamples doesn't match SamplesPerPixel. "
+ "Defining non-color channels as ExtraSamples.");
+
+ old_extrasamples = tif->tif_dir.td_extrasamples;
+ tif->tif_dir.td_extrasamples = (tif->tif_dir.td_samplesperpixel - color_channels);
+
+ // sampleinfo should contain information relative to these new extra samples
+ new_sampleinfo = (uint16*) _TIFFcalloc(tif->tif_dir.td_extrasamples, sizeof(uint16));
+ if (!new_sampleinfo) {
+ TIFFErrorExt(tif->tif_clientdata, module, "Failed to allocate memory for "
+ "temporary new sampleinfo array (%d 16 bit elements)",
+ tif->tif_dir.td_extrasamples);
+ goto bad;
+ }
+
+ memcpy(new_sampleinfo, tif->tif_dir.td_sampleinfo, old_extrasamples * sizeof(uint16));
+ _TIFFsetShortArray(&tif->tif_dir.td_sampleinfo, new_sampleinfo, tif->tif_dir.td_extrasamples);
+ _TIFFfree(new_sampleinfo);
+ }
+
/*
* Verify Palette image has a Colormap.
*/
diff --git a/libtiff/tif_print.c b/libtiff/tif_print.c
index 8deceb2..1d86adb 100644
--- a/libtiff/tif_print.c
+++ b/libtiff/tif_print.c
@@ -546,7 +546,7 @@ TIFFPrintDirectory(TIFF* tif, FILE* fd, long flags)
uint16 i;
fprintf(fd, " %2ld: %5u",
l, td->td_transferfunction[0][l]);
- for (i = 1; i < td->td_samplesperpixel; i++)
+ for (i = 1; i < td->td_samplesperpixel - td->td_extrasamples && i < 3; i++)
fprintf(fd, " %5u",
td->td_transferfunction[i][l]);
fputc('\n', fd);
--
libgit2 0.27.0
From 9171da596c88e6a2dadcab4a3a89dddd6e1b4655 Mon Sep 17 00:00:00 2001
From: Nathan Baker <elitebadger@gmail.com>
Date: Thu, 25 Jan 2018 21:28:15 +0000
Subject: [PATCH] Add workaround to pal2rgb buffer overflow.
---
tools/pal2rgb.c | 17 +++++++++++++++--
1 file changed, 15 insertions(+), 2 deletions(-)
diff --git a/tools/pal2rgb.c b/tools/pal2rgb.c
index 0423598..01fcf94 100644
--- a/tools/pal2rgb.c
+++ b/tools/pal2rgb.c
@@ -184,8 +184,21 @@ main(int argc, char* argv[])
{ unsigned char *ibuf, *obuf;
register unsigned char* pp;
register uint32 x;
- ibuf = (unsigned char*)_TIFFmalloc(TIFFScanlineSize(in));
- obuf = (unsigned char*)_TIFFmalloc(TIFFScanlineSize(out));
+ tmsize_t tss_in = TIFFScanlineSize(in);
+ tmsize_t tss_out = TIFFScanlineSize(out);
+ if (tss_out / tss_in < 3) {
+ /*
+ * BUG 2750: The following code does not know about chroma
+ * subsampling of JPEG data. It assumes that the output buffer is 3x
+ * the length of the input buffer due to exploding the palette into
+ * RGB tuples. If this assumption is incorrect, it could lead to a
+ * buffer overflow. Go ahead and fail now to prevent that.
+ */
+ fprintf(stderr, "Could not determine correct image size for output. Exiting.\n");
+ return -1;
+ }
+ ibuf = (unsigned char*)_TIFFmalloc(tss_in);
+ obuf = (unsigned char*)_TIFFmalloc(tss_out);
switch (config) {
case PLANARCONFIG_CONTIG:
for (row = 0; row < imagelength; row++) {
--
libgit2 0.27.0
From 681748ec2f5ce88da5f9fa6831e1653e46af8a66 Mon Sep 17 00:00:00 2001
From: Even Rouault <even.rouault@spatialys.com>
Date: Sun, 14 Oct 2018 16:38:29 +0200
Subject: [PATCH] JBIG: fix potential out-of-bounds write in JBIGDecode()
JBIGDecode doesn't check if the user provided buffer is large enough
to store the JBIG decoded image, which can potentially cause out-of-bounds
write in the buffer.
This issue was reported and analyzed by Thomas Dullien.
Also fixes a (harmless) potential use of uninitialized memory when
tif->tif_rawsize > tif->tif_rawcc
And in case libtiff is compiled with CHUNKY_STRIP_READ_SUPPORT, make sure
that whole strip data is provided to JBIGDecode()
Index: tiff-4.0.8/libtiff/tif_jbig.c
===================================================================
--- tiff-4.0.8.orig/libtiff/tif_jbig.c 2018-10-25 15:35:44.812533616 +0200
+++ tiff-4.0.8/libtiff/tif_jbig.c 2018-10-25 15:35:44.804533616 +0200
@@ -53,17 +53,18 @@
struct jbg_dec_state decoder;
int decodeStatus = 0;
unsigned char* pImage = NULL;
- (void) size, (void) s;
+ unsigned long decodedSize;
+ (void) s;
if (isFillOrder(tif, tif->tif_dir.td_fillorder))
{
- TIFFReverseBits(tif->tif_rawdata, tif->tif_rawdatasize);
+ TIFFReverseBits(tif->tif_rawcp, tif->tif_rawcc);
}
jbg_dec_init(&decoder);
#if defined(HAVE_JBG_NEWLEN)
- jbg_newlen(tif->tif_rawdata, (size_t)tif->tif_rawdatasize);
+ jbg_newlen(tif->tif_rawcp, (size_t)tif->tif_rawcc);
/*
* I do not check the return status of jbg_newlen because even if this
* function fails it does not necessarily mean that decoding the image
@@ -76,8 +77,8 @@
*/
#endif /* HAVE_JBG_NEWLEN */
- decodeStatus = jbg_dec_in(&decoder, (unsigned char*)tif->tif_rawdata,
- (size_t)tif->tif_rawdatasize, NULL);
+ decodeStatus = jbg_dec_in(&decoder, (unsigned char*)tif->tif_rawcp,
+ (size_t)tif->tif_rawcc, NULL);
if (JBG_EOK != decodeStatus)
{
/*
@@ -98,9 +99,28 @@
return 0;
}
+ decodedSize = jbg_dec_getsize(&decoder);
+ if( (tmsize_t)decodedSize < size )
+ {
+ TIFFWarningExt(tif->tif_clientdata, "JBIG",
+ "Only decoded %lu bytes, whereas %lu requested",
+ decodedSize, (unsigned long)size);
+ }
+ else if( (tmsize_t)decodedSize > size )
+ {
+ TIFFErrorExt(tif->tif_clientdata, "JBIG",
+ "Decoded %lu bytes, whereas %lu were requested",
+ decodedSize, (unsigned long)size);
+ jbg_dec_free(&decoder);
+ return 0;
+ }
pImage = jbg_dec_getimage(&decoder, 0);
- _TIFFmemcpy(buffer, pImage, jbg_dec_getsize(&decoder));
+ _TIFFmemcpy(buffer, pImage, decodedSize);
jbg_dec_free(&decoder);
+
+ tif->tif_rawcp += tif->tif_rawcc;
+ tif->tif_rawcc = 0;
+
return 1;
}
Index: tiff-4.0.8/libtiff/tif_read.c
===================================================================
--- tiff-4.0.8.orig/libtiff/tif_read.c 2018-10-25 15:30:38.184542808 +0200
+++ tiff-4.0.8/libtiff/tif_read.c 2018-10-25 15:36:32.076532199 +0200
@@ -329,6 +329,12 @@
return 0;
whole_strip = tif->tif_dir.td_stripbytecount[strip] < 10
|| isMapped(tif);
+ if( td->td_compression == COMPRESSION_JBIG )
+ {
+ /* Ideally plugins should have a way to declare they don't support
+ * chunk strip */
+ whole_strip = 1;
+ }
#else
whole_strip = 1;
#endif
From 473851d211cf8805a161820337ca74cc9615d6ef Mon Sep 17 00:00:00 2001
From: Nathan Baker <nathanb@lenovo-chrome.com>
Date: Tue, 6 Feb 2018 10:13:57 -0500
Subject: [PATCH] Fix for bug 2772
It is possible to craft a TIFF document where the IFD list is circular,
leading to an infinite loop while traversing the chain. The libtiff
directory reader has a failsafe that will break out of this loop after
reading 65535 directory entries, but it will continue processing,
consuming time and resources to process what is essentially a bogus TIFF
document.
This change fixes the above behavior by breaking out of processing when
a TIFF document has >= 65535 directories and terminating with an error.
---
contrib/addtiffo/tif_overview.c | 14 +++++++++++++-
tools/tiff2pdf.c | 10 ++++++++++
tools/tiffcrop.c | 13 +++++++++++--
3 files changed, 34 insertions(+), 3 deletions(-)
Index: tiff-4.0.8/contrib/addtiffo/tif_overview.c
===================================================================
--- tiff-4.0.8.orig/contrib/addtiffo/tif_overview.c 2018-10-28 12:17:34.733012009 +0100
+++ tiff-4.0.8/contrib/addtiffo/tif_overview.c 2018-10-28 12:17:34.721012009 +0100
@@ -65,6 +65,8 @@
# define MAX(a,b) ((a>b) ? a : b)
#endif
+#define TIFF_DIR_MAX 65534
+
void TIFFBuildOverviews( TIFF *, int, int *, int, const char *,
int (*)(double,void*), void * );
@@ -91,6 +93,7 @@
{
toff_t nBaseDirOffset;
toff_t nOffset;
+ tdir_t iNumDir;
(void) bUseSubIFDs;
@@ -147,7 +150,16 @@
return 0;
TIFFWriteDirectory( hTIFF );
- TIFFSetDirectory( hTIFF, (tdir_t) (TIFFNumberOfDirectories(hTIFF)-1) );
+ iNumDir = TIFFNumberOfDirectories(hTIFF);
+ if( iNumDir > TIFF_DIR_MAX )
+ {
+ TIFFErrorExt( TIFFClientdata(hTIFF),
+ "TIFF_WriteOverview",
+ "File `%s' has too many directories.\n",
+ TIFFFileName(hTIFF) );
+ exit(-1);
+ }
+ TIFFSetDirectory( hTIFF, (tdir_t) (iNumDir - 1) );
nOffset = TIFFCurrentDirOffset( hTIFF );
Index: tiff-4.0.8/tools/tiff2pdf.c
===================================================================
--- tiff-4.0.8.orig/tools/tiff2pdf.c 2018-10-28 12:17:34.733012009 +0100
+++ tiff-4.0.8/tools/tiff2pdf.c 2018-10-28 12:19:07.665009223 +0100
@@ -68,6 +68,8 @@
#define PS_UNIT_SIZE 72.0F
+#define TIFF_DIR_MAX 65534
+
/* This type is of PDF color spaces. */
typedef enum {
T2P_CS_BILEVEL = 0x01, /* Bilevel, black and white */
@@ -1051,6 +1053,14 @@
uint16* tiff_transferfunction[3];
directorycount=TIFFNumberOfDirectories(input);
+ if(directorycount > TIFF_DIR_MAX) {
+ TIFFError(
+ TIFF2PDF_MODULE,
+ "TIFF contains too many directories, %s",
+ TIFFFileName(input));
+ t2p->t2p_error = T2P_ERR_ERROR;
+ return;
+ }
t2p->tiff_pages = (T2P_PAGE*) _TIFFmalloc(TIFFSafeMultiply(tmsize_t,directorycount,sizeof(T2P_PAGE)));
if(t2p->tiff_pages==NULL){
TIFFError(
Index: tiff-4.0.8/tools/tiffcrop.c
===================================================================
--- tiff-4.0.8.orig/tools/tiffcrop.c 2018-10-28 12:17:34.733012009 +0100
+++ tiff-4.0.8/tools/tiffcrop.c 2018-10-28 12:17:34.725012009 +0100
@@ -217,6 +217,8 @@
#define DUMP_TEXT 1
#define DUMP_RAW 2
+#define TIFF_DIR_MAX 65534
+
/* Offsets into buffer for margins and fixed width and length segments */
struct offset {
uint32 tmargin;
@@ -2233,7 +2235,7 @@
pageNum = -1;
else
total_images = 0;
- /* read multiple input files and write to output file(s) */
+ /* Read multiple input files and write to output file(s) */
while (optind < argc - 1)
{
in = TIFFOpen (argv[optind], "r");
@@ -2241,7 +2243,14 @@
return (-3);
/* If only one input file is specified, we can use directory count */
- total_images = TIFFNumberOfDirectories(in);
+ total_images = TIFFNumberOfDirectories(in);
+ if (total_images > TIFF_DIR_MAX)
+ {
+ TIFFError (TIFFFileName(in), "File contains too many directories");
+ if (out != NULL)
+ (void) TIFFClose(out);
+ return (1);
+ }
if (image_count == 0)
{
dirnum = 0;
From 58a898cb4459055bb488ca815c23b880c242a27d Mon Sep 17 00:00:00 2001
From: Even Rouault <even.rouault@spatialys.com>
Date: Sat, 12 May 2018 15:32:31 +0200
Subject: [PATCH] LZWDecodeCompat(): fix potential index-out-of-bounds write. Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2780 / CVE-2018-8905
The fix consists in using the similar code LZWDecode() to validate we
don't write outside of the output buffer.
libtiff/tif_lzw.c | 17 ++++++++++++-----
1 file changed, 12 insertions(+), 5 deletions(-)
Index: tiff-4.0.8/libtiff/tif_lzw.c
===================================================================
--- tiff-4.0.8.orig/libtiff/tif_lzw.c 2018-10-28 12:59:52.864935922 +0100
+++ tiff-4.0.8/libtiff/tif_lzw.c 2018-10-28 13:01:50.136932407 +0100
@@ -603,6 +603,7 @@
char *tp;
unsigned char *bp;
int code, nbits;
+ int len;
long nextbits, nextdata, nbitsmask;
code_t *codep, *free_entp, *maxcodep, *oldcodep;
@@ -751,13 +752,18 @@
} while (--occ);
break;
}
- assert(occ >= codep->length);
- op += codep->length;
- occ -= codep->length;
- tp = op;
+ len = codep->length;
+ tp = op + len;
do {
- *--tp = codep->value;
- } while( (codep = codep->next) != NULL );
+ int t;
+ --tp;
+ t = codep->value;
+ codep = codep->next;
+ *tp = (char)t;
+ } while (codep && tp > op);
+ assert(occ >= len);
+ op += len;
+ occ -= len;
} else {
*op++ = (char)code;
occ--;
From f1b94e8a3ba49febdd3361c0214a1d1149251577 Mon Sep 17 00:00:00 2001
From: Young_X <YangX92@hotmail.com>
Date: Sat, 8 Sep 2018 14:36:12 +0800
Subject: [PATCH] only read/write TIFFTAG_GROUP3OPTIONS or
TIFFTAG_GROUP4OPTIONS if compression is COMPRESSION_CCITTFAX3 or
COMPRESSION_CCITTFAX4
---
tools/pal2rgb.c | 18 +++++++++++++++++-
tools/tiff2bw.c | 18 +++++++++++++++++-
2 files changed, 34 insertions(+), 2 deletions(-)
diff --git a/tools/pal2rgb.c b/tools/pal2rgb.c
index 01fcf941..01d8502e 100644
--- a/tools/pal2rgb.c
+++ b/tools/pal2rgb.c
@@ -402,7 +402,23 @@ cpTags(TIFF* in, TIFF* out)
{
struct cpTag *p;
for (p = tags; p < &tags[NTAGS]; p++)
- cpTag(in, out, p->tag, p->count, p->type);
+ {
+ if( p->tag == TIFFTAG_GROUP3OPTIONS )
+ {
+ uint16 compression;
+ if( !TIFFGetField(in, TIFFTAG_COMPRESSION, &compression) ||
+ compression != COMPRESSION_CCITTFAX3 )
+ continue;
+ }
+ if( p->tag == TIFFTAG_GROUP4OPTIONS )
+ {
+ uint16 compression;
+ if( !TIFFGetField(in, TIFFTAG_COMPRESSION, &compression) ||
+ compression != COMPRESSION_CCITTFAX4 )
+ continue;
+ }
+ cpTag(in, out, p->tag, p->count, p->type);
+ }
}
#undef NTAGS
diff --git a/tools/tiff2bw.c b/tools/tiff2bw.c
index 05faba87..5bef3142 100644
--- a/tools/tiff2bw.c
+++ b/tools/tiff2bw.c
@@ -450,7 +450,23 @@ cpTags(TIFF* in, TIFF* out)
{
struct cpTag *p;
for (p = tags; p < &tags[NTAGS]; p++)
- cpTag(in, out, p->tag, p->count, p->type);
+ {
+ if( p->tag == TIFFTAG_GROUP3OPTIONS )
+ {
+ uint16 compression;
+ if( !TIFFGetField(in, TIFFTAG_COMPRESSION, &compression) ||
+ compression != COMPRESSION_CCITTFAX3 )
+ continue;
+ }
+ if( p->tag == TIFFTAG_GROUP4OPTIONS )
+ {
+ uint16 compression;
+ if( !TIFFGetField(in, TIFFTAG_COMPRESSION, &compression) ||
+ compression != COMPRESSION_CCITTFAX4 )
+ continue;
+ }
+ cpTag(in, out, p->tag, p->count, p->type);
+ }
}
#undef NTAGS
--
2.18.1
From de144fd228e4be8aa484c3caf3d814b6fa88c6d9 Mon Sep 17 00:00:00 2001
From: Even Rouault <even.rouault@spatialys.com>
Date: Sat, 12 May 2018 14:24:15 +0200
Subject: [PATCH] TIFFWriteDirectorySec: avoid assertion. Fixes
http://bugzilla.maptools.org/show_bug.cgi?id=2795. CVE-2018-10963
---
libtiff/tif_dirwrite.c | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/libtiff/tif_dirwrite.c b/libtiff/tif_dirwrite.c
index 2430de6d..c15a28db 100644
--- a/libtiff/tif_dirwrite.c
+++ b/libtiff/tif_dirwrite.c
@@ -695,8 +695,11 @@ TIFFWriteDirectorySec(TIFF* tif, int isimage, int imagedone, uint64* pdiroff)
}
break;
default:
- assert(0); /* we should never get here */
- break;
+ TIFFErrorExt(tif->tif_clientdata,module,
+ "Cannot write tag %d (%s)",
+ TIFFFieldTag(o),
+ o->field_name ? o->field_name : "unknown");
+ goto bad;
}
}
}
--
2.18.1
diff --git a/tools/tiffcp.c b/tools/tiffcp.c
index 2f406e2d7065d31e2fccee0c08871cdab1d07fb7..8c81aa4f2d3adb332d00b317c887842f9c9f2b52 100644
--- a/tools/tiffcp.c
+++ b/tools/tiffcp.c
@@ -43,6 +43,7 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
+#include <limits.h>
#include <ctype.h>
@@ -1391,7 +1392,7 @@ DECLAREreadFunc(readSeparateTilesIntoBuffer)
int status = 1;
uint32 imagew = TIFFRasterScanlineSize(in);
uint32 tilew = TIFFTileRowSize(in);
- int iskew = imagew - tilew*spp;
+ int iskew;
tsize_t tilesize = TIFFTileSize(in);
tdata_t tilebuf;
uint8* bufp = (uint8*) buf;
@@ -1399,6 +1400,12 @@ DECLAREreadFunc(readSeparateTilesIntoBuffer)
uint32 row;
uint16 bps = 0, bytes_per_sample;
+ if (spp > (INT_MAX / tilew))
+ {
+ TIFFError(TIFFFileName(in), "Error, cannot handle that much samples per tile row (Tile Width * Samples/Pixel)");
+ return 0;
+ }
+ iskew = imagew - tilew*spp;
tilebuf = _TIFFmalloc(tilesize);
if (tilebuf == 0)
return 0;
diff --git a/libtiff/tif_dirwrite.c b/libtiff/tif_dirwrite.c
index c15a28dbd8fcb99b81fa5a1d44fcbcda881f42a7..ef30c869d30e210d90be16ce91f44087925fbad3 100644
--- a/libtiff/tif_dirwrite.c
+++ b/libtiff/tif_dirwrite.c
@@ -1895,12 +1895,14 @@ TIFFWriteDirectoryTagTransferfunction(TIFF* tif, uint32* ndir, TIFFDirEntry* dir
n=3;
if (n==3)
{
- if (!_TIFFmemcmp(tif->tif_dir.td_transferfunction[0],tif->tif_dir.td_transferfunction[2],m*sizeof(uint16)))
+ if (tif->tif_dir.td_transferfunction[2] == NULL ||
+ !_TIFFmemcmp(tif->tif_dir.td_transferfunction[0],tif->tif_dir.td_transferfunction[2],m*sizeof(uint16)))
n=2;
}
if (n==2)
{
- if (!_TIFFmemcmp(tif->tif_dir.td_transferfunction[0],tif->tif_dir.td_transferfunction[1],m*sizeof(uint16)))
+ if (tif->tif_dir.td_transferfunction[1] == NULL ||
+ !_TIFFmemcmp(tif->tif_dir.td_transferfunction[0],tif->tif_dir.td_transferfunction[1],m*sizeof(uint16)))
n=1;
}
if (n==0)
diff --git a/tools/ppm2tiff.c b/tools/ppm2tiff.c
index af6e41243bb4edbc229af473222964b2dac2e75c..c2d59257b9e7847719281e3ec8559a0359767b12 100644
--- a/tools/ppm2tiff.c
+++ b/tools/ppm2tiff.c
@@ -72,15 +72,16 @@ BadPPM(char* file)
exit(-2);
}
+
+#define TIFF_SIZE_T_MAX ((size_t) ~ ((size_t)0))
+#define TIFF_TMSIZE_T_MAX (tmsize_t)(TIFF_SIZE_T_MAX >> 1)
+
static tmsize_t
multiply_ms(tmsize_t m1, tmsize_t m2)
{
- tmsize_t bytes = m1 * m2;
-
- if (m1 && bytes / m1 != m2)
- bytes = 0;
-
- return bytes;
+ if( m1 == 0 || m2 > TIFF_TMSIZE_T_MAX / m1 )
+ return 0;
+ return m1 * m2;
}
int
diff --git a/libtiff/tif_dir.c b/libtiff/tif_dir.c
index 6f0b48798bdeee91729c10e1fbcf9786234be5f3..078fbcec20677f19f7f967a4834011fe60df1df3 100644
--- a/libtiff/tif_dir.c
+++ b/libtiff/tif_dir.c
@@ -287,6 +287,18 @@ _TIFFVSetField(TIFF* tif, uint32 tag, va_list ap)
_TIFFfree(td->td_smaxsamplevalue);
td->td_smaxsamplevalue = NULL;
}
+ /* Test if 3 transfer functions instead of just one are now needed
+ See http://bugzilla.maptools.org/show_bug.cgi?id=2820 */
+ if( td->td_transferfunction[0] != NULL && (v - td->td_extrasamples > 1) &&
+ !(td->td_samplesperpixel - td->td_extrasamples > 1))
+ {
+ TIFFWarningExt(tif->tif_clientdata,module,
+ "SamplesPerPixel tag value is changing, "
+ "but TransferFunction was read with a different value. Cancelling it");
+ TIFFClrFieldBit(tif,FIELD_TRANSFERFUNCTION);
+ _TIFFfree(td->td_transferfunction[0]);
+ td->td_transferfunction[0] = NULL;
+ }
}
td->td_samplesperpixel = (uint16) v;
break;
@@ -363,6 +375,16 @@ _TIFFVSetField(TIFF* tif, uint32 tag, va_list ap)
_TIFFsetShortArray(&td->td_colormap[2], va_arg(ap, uint16*), v32);
break;
case TIFFTAG_EXTRASAMPLES:
+ if ( td->td_transferfunction[0] != NULL && (td->td_samplesperpixel - v > 1) &&
+ !(td->td_samplesperpixel - td->td_extrasamples > 1))
+ {
+ TIFFWarningExt(tif->tif_clientdata,module,
+ "ExtraSamples tag value is changing, "
+ "but TransferFunction was read with a different value. Cancelling it");
+ TIFFClrFieldBit(tif,FIELD_TRANSFERFUNCTION);
+ _TIFFfree(td->td_transferfunction[0]);
+ td->td_transferfunction[0] = NULL;
+ }
if (!setExtraSamples(td, ap, &v))
goto badvalue;
break;
diff --git a/libtiff/tif_dir.c b/libtiff/tif_dir.c
index 078fbcec20677f19f7f967a4834011fe60df1df3..028ea54a256b4123ac320138aaedd1b356c2132f 100644
--- a/libtiff/tif_dir.c
+++ b/libtiff/tif_dir.c
@@ -90,13 +90,15 @@ setDoubleArrayOneValue(double** vpp, double value, size_t nmemb)
* Install extra samples information.
*/
static int
-setExtraSamples(TIFFDirectory* td, va_list ap, uint32* v)
+setExtraSamples(TIFF* tif, va_list ap, uint32* v)
{
/* XXX: Unassociated alpha data == 999 is a known Corel Draw bug, see below */
#define EXTRASAMPLE_COREL_UNASSALPHA 999
uint16* va;
uint32 i;
+ TIFFDirectory* td = &tif->tif_dir;
+ static const char module[] = "setExtraSamples";
*v = (uint16) va_arg(ap, uint16_vap);
if ((uint16) *v > td->td_samplesperpixel)
@@ -118,6 +120,18 @@ setExtraSamples(TIFFDirectory* td, va_list ap, uint32* v)
return 0;
}
}
+
+ if ( td->td_transferfunction[0] != NULL && (td->td_samplesperpixel - *v > 1) &&
+ !(td->td_samplesperpixel - td->td_extrasamples > 1))
+ {
+ TIFFWarningExt(tif->tif_clientdata,module,
+ "ExtraSamples tag value is changing, "
+ "but TransferFunction was read with a different value. Cancelling it");
+ TIFFClrFieldBit(tif,FIELD_TRANSFERFUNCTION);
+ _TIFFfree(td->td_transferfunction[0]);
+ td->td_transferfunction[0] = NULL;
+ }
+
td->td_extrasamples = (uint16) *v;
_TIFFsetShortArray(&td->td_sampleinfo, va, td->td_extrasamples);
return 1;
@@ -375,17 +389,7 @@ _TIFFVSetField(TIFF* tif, uint32 tag, va_list ap)
_TIFFsetShortArray(&td->td_colormap[2], va_arg(ap, uint16*), v32);
break;
case TIFFTAG_EXTRASAMPLES:
- if ( td->td_transferfunction[0] != NULL && (td->td_samplesperpixel - v > 1) &&
- !(td->td_samplesperpixel - td->td_extrasamples > 1))
- {
- TIFFWarningExt(tif->tif_clientdata,module,
- "ExtraSamples tag value is changing, "
- "but TransferFunction was read with a different value. Cancelling it");
- TIFFClrFieldBit(tif,FIELD_TRANSFERFUNCTION);
- _TIFFfree(td->td_transferfunction[0]);
- td->td_transferfunction[0] = NULL;
- }
- if (!setExtraSamples(td, ap, &v))
+ if (!setExtraSamples(tif, ap, &v))
goto badvalue;
break;
case TIFFTAG_MATTEING:
diff --git a/libtiff/tif_aux.c b/libtiff/tif_aux.c
index 90d30214c6f50d60b5a896d644942cb04ff6e8ea..3e9bda43d5f8b20274ad306499bb85c5a06c73e9 100644
--- a/libtiff/tif_aux.c
+++ b/libtiff/tif_aux.c
@@ -59,18 +59,57 @@ _TIFFMultiply64(TIFF* tif, uint64 first, uint64 second, const char* where)
return bytes;
}
+tmsize_t
+_TIFFMultiplySSize(TIFF* tif, tmsize_t first, tmsize_t second, const char* where)
+{
+ if( first <= 0 || second <= 0 )
+ {
+ if( tif != NULL && where != NULL )
+ {
+ TIFFErrorExt(tif->tif_clientdata, where,
+ "Invalid argument to _TIFFMultiplySSize() in %s", where);
+ }
+ return 0;
+ }
+
+ if( first > TIFF_TMSIZE_T_MAX / second )
+ {
+ if( tif != NULL && where != NULL )
+ {
+ TIFFErrorExt(tif->tif_clientdata, where,
+ "Integer overflow in %s", where);
+ }
+ return 0;
+ }
+ return first * second;
+}
+
+tmsize_t _TIFFCastUInt64ToSSize(TIFF* tif, uint64 val, const char* module)
+{
+ if( val > (uint64)TIFF_TMSIZE_T_MAX )
+ {
+ if( tif != NULL && module != NULL )
+ {
+ TIFFErrorExt(tif->tif_clientdata,module,"Integer overflow");
+ }
+ return 0;
+ }
+ return (tmsize_t)val;
+}
+
void*
_TIFFCheckRealloc(TIFF* tif, void* buffer,
tmsize_t nmemb, tmsize_t elem_size, const char* what)
{
void* cp = NULL;
- tmsize_t bytes = nmemb * elem_size;
-
+ tmsize_t count = _TIFFMultiplySSize(tif, nmemb, elem_size, NULL);
/*
- * XXX: Check for integer overflow.
+ * Check for integer overflow.
*/
- if (nmemb && elem_size && bytes / elem_size == nmemb)
- cp = _TIFFrealloc(buffer, bytes);
+ if (count != 0)
+ {
+ cp = _TIFFrealloc(buffer, count);
+ }
if (cp == NULL) {
TIFFErrorExt(tif->tif_clientdata, tif->tif_name,
diff --git a/libtiff/tif_getimage.c b/libtiff/tif_getimage.c
index 599bf12798044f0ff49405de30300fe68f1d17e3..c88b5fa67dc640e1849f9b33bfbc887d0a66d15d 100644
--- a/libtiff/tif_getimage.c
+++ b/libtiff/tif_getimage.c
@@ -755,9 +755,8 @@ gtTileSeparate(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
uint32 leftmost_tw;
tilesize = TIFFTileSize(tif);
- bufsize = TIFFSafeMultiply(tmsize_t,alpha?4:3,tilesize);
+ bufsize = _TIFFMultiplySSize(tif, alpha?4:3,tilesize, "gtTileSeparate");
if (bufsize == 0) {
- TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif), "Integer overflow in %s", "gtTileSeparate");
return (0);
}
buf = (unsigned char*) _TIFFmalloc(bufsize);
@@ -1006,9 +1005,8 @@ gtStripSeparate(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
uint16 colorchannels;
stripsize = TIFFStripSize(tif);
- bufsize = TIFFSafeMultiply(tmsize_t,alpha?4:3,stripsize);
+ bufsize = _TIFFMultiplySSize(tif,alpha?4:3,stripsize, "gtStripSeparate");
if (bufsize == 0) {
- TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif), "Integer overflow in %s", "gtStripSeparate");
return (0);
}
diff --git a/libtiff/tif_luv.c b/libtiff/tif_luv.c
index 6a63eadcab341f1c70cd1d34c27713da8ca0dd76..6fe485884ceb73e48985bc02d620ae7ac3a7eed0 100644
--- a/libtiff/tif_luv.c
+++ b/libtiff/tif_luv.c
@@ -1264,16 +1264,10 @@ LogL16GuessDataFmt(TIFFDirectory *td)
return (SGILOGDATAFMT_UNKNOWN);
}
-
-#define TIFF_SIZE_T_MAX ((size_t) ~ ((size_t)0))
-#define TIFF_TMSIZE_T_MAX (tmsize_t)(TIFF_SIZE_T_MAX >> 1)
-
static tmsize_t
multiply_ms(tmsize_t m1, tmsize_t m2)
{
- if( m1 == 0 || m2 > TIFF_TMSIZE_T_MAX / m1 )
- return 0;
- return m1 * m2;
+ return _TIFFMultiplySSize(NULL, m1, m2, NULL);
}
static int
diff --git a/libtiff/tif_pixarlog.c b/libtiff/tif_pixarlog.c
index b1e48d99c9246191253f86083b7046a316ec5171..a2149c852bc425c08677081d20d6eff81755cb95 100644
--- a/libtiff/tif_pixarlog.c
+++ b/libtiff/tif_pixarlog.c
@@ -636,15 +636,10 @@ PixarLogGuessDataFmt(TIFFDirectory *td)
return guess;
}
-#define TIFF_SIZE_T_MAX ((size_t) ~ ((size_t)0))
-#define TIFF_TMSIZE_T_MAX (tmsize_t)(TIFF_SIZE_T_MAX >> 1)
-
static tmsize_t
multiply_ms(tmsize_t m1, tmsize_t m2)
{
- if( m1 == 0 || m2 > TIFF_TMSIZE_T_MAX / m1 )
- return 0;
- return m1 * m2;
+ return _TIFFMultiplySSize(NULL, m1, m2, NULL);
}
static tmsize_t
diff --git a/libtiff/tif_read.c b/libtiff/tif_read.c
index 1f5362a7a4269821ada45eb31ec597d65508c592..c436b0d5ecc2f580d542d5d0bd58cf513b7896c7 100644
--- a/libtiff/tif_read.c
+++ b/libtiff/tif_read.c
@@ -31,9 +31,6 @@
#include "tiffiop.h"
#include <stdio.h>
-#define TIFF_SIZE_T_MAX ((size_t) ~ ((size_t)0))
-#define TIFF_TMSIZE_T_MAX (tmsize_t)(TIFF_SIZE_T_MAX >> 1)
-
int TIFFFillStrip(TIFF* tif, uint32 strip);
int TIFFFillTile(TIFF* tif, uint32 tile);
static int TIFFStartStrip(TIFF* tif, uint32 strip);
@@ -51,6 +48,8 @@ TIFFReadRawTile1(TIFF* tif, uint32 tile, void* buf, tmsize_t size, const char* m
#define THRESHOLD_MULTIPLIER 10
#define MAX_THRESHOLD (THRESHOLD_MULTIPLIER * THRESHOLD_MULTIPLIER * THRESHOLD_MULTIPLIER * INITIAL_THRESHOLD)
+#define TIFF_INT64_MAX ((((int64)0x7FFFFFFF) << 32) | 0xFFFFFFFF)
+
/* Read 'size' bytes in tif_rawdata buffer starting at offset 'rawdata_offset'
* Returns 1 in case of success, 0 otherwise. */
static int TIFFReadAndRealloc( TIFF* tif, tmsize_t size,
@@ -716,23 +716,8 @@ TIFFReadRawStrip(TIFF* tif, uint32 strip, void* buf, tmsize_t size)
return ((tmsize_t)(-1));
}
bytecount = td->td_stripbytecount[strip];
- if ((int64)bytecount <= 0) {
-#if defined(__WIN32__) && (defined(_MSC_VER) || defined(__MINGW32__))
- TIFFErrorExt(tif->tif_clientdata, module,
- "%I64u: Invalid strip byte count, strip %lu",
- (unsigned __int64) bytecount,
- (unsigned long) strip);
-#else
- TIFFErrorExt(tif->tif_clientdata, module,
- "%llu: Invalid strip byte count, strip %lu",
- (unsigned long long) bytecount,
- (unsigned long) strip);
-#endif
- return ((tmsize_t)(-1));
- }
- bytecountm = (tmsize_t)bytecount;
- if ((uint64)bytecountm!=bytecount) {
- TIFFErrorExt(tif->tif_clientdata, module, "Integer overflow");
+ bytecountm = _TIFFCastUInt64ToSSize(tif, bytecount, module);
+ if (bytecountm == 0) {
return ((tmsize_t)(-1));
}
if (size != (tmsize_t)(-1) && size < bytecountm)
@@ -756,7 +756,7 @@ TIFFFillStrip(TIFF* tif, uint32 strip)
if ((tif->tif_flags&TIFF_NOREADRAW)==0)
{
uint64 bytecount = td->td_stripbytecount[strip];
- if ((int64)bytecount <= 0) {
+ if( bytecount == 0 || bytecount > (uint64)TIFF_INT64_MAX ) {
#if defined(__WIN32__) && (defined(_MSC_VER) || defined(__MINGW32__))
TIFFErrorExt(tif->tif_clientdata, module,
"Invalid strip byte count %I64u, strip %lu",
@@ -783,7 +783,7 @@ TIFFFillStrip(TIFF* tif, uint32 strip)
(bytecount - 4096) / 10 > (uint64)stripsize )
{
uint64 newbytecount = (uint64)stripsize * 10 + 4096;
- if( (int64)newbytecount >= 0 )
+ if( newbytecount == 0 || newbytecount > (uint64)TIFF_INT64_MAX )
{
#if defined(__WIN32__) && (defined(_MSC_VER) || defined(__MINGW32__))
TIFFWarningExt(tif->tif_clientdata, module,
@@ -1104,10 +1104,8 @@ TIFFReadRawTile(TIFF* tif, uint32 tile,
bytecount64 = td->td_stripbytecount[tile];
if (size != (tmsize_t)(-1) && (uint64)size < bytecount64)
bytecount64 = (uint64)size;
- bytecountm = (tmsize_t)bytecount64;
- if ((uint64)bytecountm!=bytecount64)
- {
- TIFFErrorExt(tif->tif_clientdata,module,"Integer overflow");
+ bytecountm = _TIFFCastUInt64ToSSize(tif, bytecount64, module);
+ if( bytecountm == 0 ) {
return ((tmsize_t)(-1));
}
return (TIFFReadRawTile1(tif, tile, buf, bytecountm, module));
@@ -1129,7 +1129,7 @@ TIFFFillTile(TIFF* tif, uint32 tile)
if ((tif->tif_flags&TIFF_NOREADRAW)==0)
{
uint64 bytecount = td->td_stripbytecount[tile];
- if ((int64)bytecount <= 0) {
+ if( bytecount == 0 || bytecount > (uint64)TIFF_INT64_MAX ) {
#if defined(__WIN32__) && (defined(_MSC_VER) || defined(__MINGW32__))
TIFFErrorExt(tif->tif_clientdata, module,
"%I64u: Invalid tile byte count, tile %lu",
@@ -1143,6 +1143,39 @@ TIFFFillTile(TIFF* tif, uint32 tile)
#endif
return (0);
}
+
+ /* To avoid excessive memory allocations: */
+ /* Byte count should normally not be larger than a number of */
+ /* times the uncompressed size plus some margin */
+ if( bytecount > 1024 * 1024 )
+ {
+ /* 10 and 4096 are just values that could be adjusted. */
+ /* Hopefully they are safe enough for all codecs */
+ tmsize_t stripsize = TIFFTileSize(tif);
+ if( stripsize != 0 &&
+ (bytecount - 4096) / 10 > (uint64)stripsize )
+ {
+ uint64 newbytecount = (uint64)stripsize * 10 + 4096;
+ if( newbytecount == 0 || newbytecount > (uint64)TIFF_INT64_MAX )
+ {
+#if defined(__WIN32__) && (defined(_MSC_VER) || defined(__MINGW32__))
+ TIFFWarningExt(tif->tif_clientdata, module,
+ "Too large tile byte count %I64u, tile %lu. Limiting to %I64u",
+ (unsigned __int64) bytecount,
+ (unsigned long) tile,
+ (unsigned __int64) newbytecount);
+#else
+ TIFFErrorExt(tif->tif_clientdata, module,
+ "Too large tile byte count %llu, tile %lu. Limiting to %llu",
+ (unsigned long long) bytecount,
+ (unsigned long) tile,
+ (unsigned long long) newbytecount);
+#endif
+ bytecount = newbytecount;
+ }
+ }
+ }
+
if (isMapped(tif) &&
(isFillOrder(tif, td->td_fillorder)
|| (tif->tif_flags & TIFF_NOBITREV))) {
diff --git a/libtiff/tif_strip.c b/libtiff/tif_strip.c
index 2f3cd883b4106f7b43ee0dcec6c47a560cf3d135..c08c60a7928d1b27a4d66b385acd46faea992603 100644
--- a/libtiff/tif_strip.c
+++ b/libtiff/tif_strip.c
@@ -131,15 +131,8 @@ TIFFVStripSize(TIFF* tif, uint32 nrows)
{
static const char module[] = "TIFFVStripSize";
uint64 m;
- tmsize_t n;
m=TIFFVStripSize64(tif,nrows);
- n=(tmsize_t)m;
- if ((uint64)n!=m)
- {
- TIFFErrorExt(tif->tif_clientdata,module,"Integer overflow");
- n=0;
- }
- return(n);
+ return _TIFFCastUInt64ToSSize(tif, m, module);
}
/*
@@ -213,15 +206,8 @@ TIFFStripSize(TIFF* tif)
{
static const char module[] = "TIFFStripSize";
uint64 m;
- tmsize_t n;
m=TIFFStripSize64(tif);
- n=(tmsize_t)m;
- if ((uint64)n!=m)
- {
- TIFFErrorExt(tif->tif_clientdata,module,"Integer overflow");
- n=0;
- }
- return(n);
+ return _TIFFCastUInt64ToSSize(tif, m, module);
}
/*
@@ -332,14 +318,8 @@ TIFFScanlineSize(TIFF* tif)
{
static const char module[] = "TIFFScanlineSize";
uint64 m;
- tmsize_t n;
m=TIFFScanlineSize64(tif);
- n=(tmsize_t)m;
- if ((uint64)n!=m) {
- TIFFErrorExt(tif->tif_clientdata,module,"Integer arithmetic overflow");
- n=0;
- }
- return(n);
+ return _TIFFCastUInt64ToSSize(tif, m, module);
}
/*
@@ -368,15 +348,8 @@ TIFFRasterScanlineSize(TIFF* tif)
{
static const char module[] = "TIFFRasterScanlineSize";
uint64 m;
- tmsize_t n;
m=TIFFRasterScanlineSize64(tif);
- n=(tmsize_t)m;
- if ((uint64)n!=m)
- {
- TIFFErrorExt(tif->tif_clientdata,module,"Integer arithmetic overflow");
- n=0;
- }
- return(n);
+ return _TIFFCastUInt64ToSSize(tif, m, module);
}
/* vim: set ts=8 sts=8 sw=8 noet: */
diff --git a/libtiff/tif_tile.c b/libtiff/tif_tile.c
index 58fe9354a387c7fa9b9d0333b54e4d4d163d4777..661cc771548bc15d5de572b4f04bb9d91c20bc40 100644
--- a/libtiff/tif_tile.c
+++ b/libtiff/tif_tile.c
@@ -183,15 +183,8 @@ TIFFTileRowSize(TIFF* tif)
{
static const char module[] = "TIFFTileRowSize";
uint64 m;
- tmsize_t n;
m=TIFFTileRowSize64(tif);
- n=(tmsize_t)m;
- if ((uint64)n!=m)
- {
- TIFFErrorExt(tif->tif_clientdata,module,"Integer overflow");
- n=0;
- }
- return(n);
+ return _TIFFCastUInt64ToSSize(tif, m, module);
}
/*
@@ -250,15 +243,8 @@ TIFFVTileSize(TIFF* tif, uint32 nrows)
{
static const char module[] = "TIFFVTileSize";
uint64 m;
- tmsize_t n;
m=TIFFVTileSize64(tif,nrows);
- n=(tmsize_t)m;
- if ((uint64)n!=m)
- {
- TIFFErrorExt(tif->tif_clientdata,module,"Integer overflow");
- n=0;
- }
- return(n);
+ return _TIFFCastUInt64ToSSize(tif, m, module);
}
/*
@@ -274,15 +260,8 @@ TIFFTileSize(TIFF* tif)
{
static const char module[] = "TIFFTileSize";
uint64 m;
- tmsize_t n;
m=TIFFTileSize64(tif);
- n=(tmsize_t)m;
- if ((uint64)n!=m)
- {
- TIFFErrorExt(tif->tif_clientdata,module,"Integer overflow");
- n=0;
- }
- return(n);
+ return _TIFFCastUInt64ToSSize(tif, m, module);
}
/*
diff --git a/libtiff/tiffiop.h b/libtiff/tiffiop.h
index dd6fb095aff046bd7b9a68532dcf3e00ad5ac02d..60ccc87b319050b08dd94a74f9a1aaaef66cdb54 100644
--- a/libtiff/tiffiop.h
+++ b/libtiff/tiffiop.h
@@ -78,6 +78,9 @@ extern int snprintf(char* str, size_t size, const char* format, ...);
#define FALSE 0
#endif
+#define TIFF_SIZE_T_MAX ((size_t) ~ ((size_t)0))
+#define TIFF_TMSIZE_T_MAX (tmsize_t)(TIFF_SIZE_T_MAX >> 1)
+
typedef struct client_info {
struct client_info *next;
void *data;
@@ -260,7 +263,7 @@ struct tiff {
#define TIFFhowmany8_64(x) (((x)&0x07)?((uint64)(x)>>3)+1:(uint64)(x)>>3)
#define TIFFroundup_64(x, y) (TIFFhowmany_64(x,y)*(y))
-/* Safe multiply which returns zero if there is an integer overflow */
+/* Safe multiply which returns zero if there is an *unsigned* integer overflow. This macro is not safe for *signed* integer types */
#define TIFFSafeMultiply(t,v,m) ((((t)(m) != (t)0) && (((t)(((v)*(m))/(m))) == (t)(v))) ? (t)((v)*(m)) : (t)0)
#define TIFFmax(A,B) ((A)>(B)?(A):(B))
@@ -359,6 +362,8 @@ extern TIFFErrorHandlerExt _TIFFerrorHandlerExt;
extern uint32 _TIFFMultiply32(TIFF*, uint32, uint32, const char*);
extern uint64 _TIFFMultiply64(TIFF*, uint64, uint64, const char*);
+extern tmsize_t _TIFFMultiplySSize(TIFF*, tmsize_t, tmsize_t, const char*);
+extern tmsize_t _TIFFCastUInt64ToSSize(TIFF*, uint64, const char*);
extern void* _TIFFCheckMalloc(TIFF*, tmsize_t, tmsize_t, const char*);
extern void* _TIFFCheckRealloc(TIFF*, void*, tmsize_t, tmsize_t, const char*);
diff --git a/libtiff/tif_getimage.c b/libtiff/tif_getimage.c
index c88b5fa67dc640e1849f9b33bfbc887d0a66d15d..4da785d3a1fc237cc95ab93a2eb0969556457b13 100644
--- a/libtiff/tif_getimage.c
+++ b/libtiff/tif_getimage.c
@@ -936,16 +936,23 @@ gtStripContig(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
fromskew = (w < imagewidth ? imagewidth - w : 0);
for (row = 0; row < h; row += nrow)
{
+ uint32 temp;
rowstoread = rowsperstrip - (row + img->row_offset) % rowsperstrip;
nrow = (row + rowstoread > h ? h - row : rowstoread);
nrowsub = nrow;
if ((nrowsub%subsamplingver)!=0)
nrowsub+=subsamplingver-nrowsub%subsamplingver;
+ temp = (row + img->row_offset)%rowsperstrip + nrowsub;
+ if( scanline > 0 && temp > (size_t)(TIFF_TMSIZE_T_MAX / scanline) )
+ {
+ TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif), "Integer overflow in gtStripContig");
+ return 0;
+ }
if (_TIFFReadEncodedStripAndAllocBuffer(tif,
TIFFComputeStrip(tif,row+img->row_offset, 0),
(void**)(&buf),
maxstripsize,
- ((row + img->row_offset)%rowsperstrip + nrowsub) * scanline)==(tmsize_t)(-1)
+ temp * scanline)==(tmsize_t)(-1)
&& (buf == NULL || img->stoponerr))
{
ret = 0;
@@ -1038,15 +1045,22 @@ gtStripSeparate(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
fromskew = (w < imagewidth ? imagewidth - w : 0);
for (row = 0; row < h; row += nrow)
{
+ uint32 temp;
rowstoread = rowsperstrip - (row + img->row_offset) % rowsperstrip;
nrow = (row + rowstoread > h ? h - row : rowstoread);
offset_row = row + img->row_offset;
+ temp = (row + img->row_offset)%rowsperstrip + nrow;
+ if( scanline > 0 && temp > (size_t)(TIFF_TMSIZE_T_MAX / scanline) )
+ {
+ TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif), "Integer overflow in gtStripSeparate");
+ return 0;
+ }
if( buf == NULL )
{
if (_TIFFReadEncodedStripAndAllocBuffer(
tif, TIFFComputeStrip(tif, offset_row, 0),
(void**) &buf, bufsize,
- ((row + img->row_offset)%rowsperstrip + nrow) * scanline)==(tmsize_t)(-1)
+ temp * scanline)==(tmsize_t)(-1)
&& (buf == NULL || img->stoponerr))
{
ret = 0;
@@ -1066,7 +1080,7 @@ gtStripSeparate(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
}
}
else if (TIFFReadEncodedStrip(tif, TIFFComputeStrip(tif, offset_row, 0),
- p0, ((row + img->row_offset)%rowsperstrip + nrow) * scanline)==(tmsize_t)(-1)
+ p0, temp * scanline)==(tmsize_t)(-1)
&& img->stoponerr)
{
ret = 0;
@@ -1074,7 +1088,7 @@ gtStripSeparate(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
}
if (colorchannels > 1
&& TIFFReadEncodedStrip(tif, TIFFComputeStrip(tif, offset_row, 1),
- p1, ((row + img->row_offset)%rowsperstrip + nrow) * scanline) == (tmsize_t)(-1)
+ p1, temp * scanline) == (tmsize_t)(-1)
&& img->stoponerr)
{
ret = 0;
@@ -1082,7 +1096,7 @@ gtStripSeparate(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
}
if (colorchannels > 1
&& TIFFReadEncodedStrip(tif, TIFFComputeStrip(tif, offset_row, 2),
- p2, ((row + img->row_offset)%rowsperstrip + nrow) * scanline) == (tmsize_t)(-1)
+ p2, temp * scanline) == (tmsize_t)(-1)
&& img->stoponerr)
{
ret = 0;
@@ -1091,7 +1105,7 @@ gtStripSeparate(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
if (alpha)
{
if (TIFFReadEncodedStrip(tif, TIFFComputeStrip(tif, offset_row, colorchannels),
- pa, ((row + img->row_offset)%rowsperstrip + nrow) * scanline)==(tmsize_t)(-1)
+ pa, temp * scanline)==(tmsize_t)(-1)
&& img->stoponerr)
{
ret = 0;
component/libtiff/tiff_4.0.8-3.debian.patch deleted 100644 → 0
View file @ 57575e34
diff -pur tiff-4.0.4/tools/tiffsplit.c tiff-4.0.4_patch/tools/tiffsplit.c
--- tiff-4.0.4/tools/tiffsplit.c 2015-05-28 15:10:26.000000000 +0200
+++ tiff-4.0.4_patch/tools/tiffsplit.c 2016-02-12 19:15:30.532005041 +0100
@@ -179,8 +179,9 @@ tiffcp(TIFF* in, TIFF* out)
TIFFSetField(out, TIFFTAG_JPEGTABLES, count, table);
}
}
+ uint32 count = 0;
CopyField(TIFFTAG_PHOTOMETRIC, shortv);
- CopyField(TIFFTAG_PREDICTOR, shortv);
+ CopyField2(TIFFTAG_PREDICTOR, count, shortv);
CopyField(TIFFTAG_THRESHHOLDING, shortv);
CopyField(TIFFTAG_FILLORDER, shortv);
CopyField(TIFFTAG_ORIENTATION, shortv);
@@ -188,7 +189,7 @@ tiffcp(TIFF* in, TIFF* out)
CopyField(TIFFTAG_MAXSAMPLEVALUE, shortv);
CopyField(TIFFTAG_XRESOLUTION, floatv);
CopyField(TIFFTAG_YRESOLUTION, floatv);
- CopyField(TIFFTAG_GROUP3OPTIONS, longv);
+ CopyField2(TIFFTAG_GROUP3OPTIONS, count, longv);
CopyField(TIFFTAG_GROUP4OPTIONS, longv);
CopyField(TIFFTAG_RESOLUTIONUNIT, shortv);
CopyField(TIFFTAG_PLANARCONFIG, shortv);
From 02669064e927074819ce1ed39aba0fccaa167717 Mon Sep 17 00:00:00 2001
From: erouault <erouault>
Date: Mon, 29 May 2017 10:12:54 +0000
Subject: [PATCH] * libtiff/tif_color.c: TIFFYCbCrToRGBInit(): stricter
clamping to avoid int32 overflow in TIFFYCbCrtoRGB(). Fixes
https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=1844 Credit to OSS Fuzz
---
ChangeLog | 7 +++++++
libtiff/tif_color.c | 6 +++---
2 files changed, 10 insertions(+), 3 deletions(-)
diff --git a/ChangeLog b/ChangeLog
index ee8d9d08..61116596 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,10 @@
+2017-05-29 Even Rouault <even.rouault at spatialys.com>
+
+ * libtiff/tif_color.c: TIFFYCbCrToRGBInit(): stricter clamping to avoid
+ int32 overflow in TIFFYCbCrtoRGB().
+ Fixes https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=1844
+ Credit to OSS Fuzz
+
2017-05-21 Bob Friesenhahn <bfriesen@simple.dallas.tx.us>
* configure.ac: libtiff 4.0.8 released.
diff --git a/libtiff/tif_color.c b/libtiff/tif_color.c
index 055ed3b2..10a5e66e 100644
--- a/libtiff/tif_color.c
+++ b/libtiff/tif_color.c
@@ -275,10 +275,10 @@ TIFFYCbCrToRGBInit(TIFFYCbCrToRGB* ycbcr, float *luma, float *refBlackWhite)
for (i = 0, x = -128; i < 256; i++, x++) {
int32 Cr = (int32)CLAMPw(Code2V(x, refBlackWhite[4] - 128.0F,
refBlackWhite[5] - 128.0F, 127),
- -128.0F * 64, 128.0F * 64);
+ -128.0F * 32, 128.0F * 32);
int32 Cb = (int32)CLAMPw(Code2V(x, refBlackWhite[2] - 128.0F,
refBlackWhite[3] - 128.0F, 127),
- -128.0F * 64, 128.0F * 64);
+ -128.0F * 32, 128.0F * 32);
ycbcr->Cr_r_tab[i] = (int32)((D1*Cr + ONE_HALF)>>SHIFT);
ycbcr->Cb_b_tab[i] = (int32)((D3*Cb + ONE_HALF)>>SHIFT);
@@ -286,7 +286,7 @@ TIFFYCbCrToRGBInit(TIFFYCbCrToRGB* ycbcr, float *luma, float *refBlackWhite)
ycbcr->Cb_g_tab[i] = D4*Cb + ONE_HALF;
ycbcr->Y_tab[i] =
(int32)CLAMPw(Code2V(x + 128, refBlackWhite[0], refBlackWhite[1], 255),
- -128.0F * 64, 128.0F * 64);
+ -128.0F * 32, 128.0F * 32);
}
}
From 468988860e0dae62ebbf991627c74bcbb4bd256f Mon Sep 17 00:00:00 2001
From: erouault <erouault>
Date: Mon, 29 May 2017 11:29:06 +0000
Subject: [PATCH] * libtiff/tif_getimage.c: initYCbCrConversion(): stricter
validation for refBlackWhite coefficients values. To avoid invalid
float->int32 conversion (when refBlackWhite[0] == 2147483648.f) Fixes
https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=1907 Credit to OSS Fuzz
---
ChangeLog | 8 ++++++++
libtiff/tif_getimage.c | 2 +-
2 files changed, 9 insertions(+), 1 deletion(-)
diff --git a/ChangeLog b/ChangeLog
index a2ddaac2..04881ba7 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,13 @@
2017-05-29 Even Rouault <even.rouault at spatialys.com>
+ * libtiff/tif_getimage.c: initYCbCrConversion(): stricter validation for
+ refBlackWhite coefficients values. To avoid invalid float->int32 conversion
+ (when refBlackWhite[0] == 2147483648.f)
+ Fixes https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=1907
+ Credit to OSS Fuzz
+
+2017-05-29 Even Rouault <even.rouault at spatialys.com>
+
* libtiff/tif_color.c: TIFFYCbCrToRGBInit(): stricter clamping to avoid
int32 overflow in TIFFYCbCrtoRGB().
Fixes https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=1844
diff --git a/libtiff/tif_getimage.c b/libtiff/tif_getimage.c
index dc373abc..a209a7a7 100644
--- a/libtiff/tif_getimage.c
+++ b/libtiff/tif_getimage.c
@@ -2241,7 +2241,7 @@ DECLARESepPutFunc(putseparate8bitYCbCr11tile)
static int isInRefBlackWhiteRange(float f)
{
- return f >= (float)(-0x7FFFFFFF + 128) && f <= (float)0x7FFFFFFF;
+ return f > (float)(-0x7FFFFFFF + 128) && f < (float)0x7FFFFFFF;
}
static int
commit 40448d58fbfad52d2dde5bd18daa30b17fe35fcd
Author: erouault <erouault>
Date: Thu Jun 1 12:44:04 2017 +0000
* libtiff/tif_dirinfo.c, tif_dirread.c: add _TIFFCheckFieldIsValidForCodec(),
and use it in TIFFReadDirectory() so as to ignore fields whose tag is a
codec-specified tag but this codec is not enabled. This avoids TIFFGetField()
to behave differently depending on whether the codec is enabled or not, and
thus can avoid stack based buffer overflows in a number of TIFF utilities
such as tiffsplit, tiffcmp, thumbnail, etc.
Patch derived from 0063-Handle-properly-CODEC-specific-tags.patch
(http://bugzilla.maptools.org/show_bug.cgi?id=2580) by Raphaël Hertzog.
Fixes:
http://bugzilla.maptools.org/show_bug.cgi?id=2580
http://bugzilla.maptools.org/show_bug.cgi?id=2693
http://bugzilla.maptools.org/show_bug.cgi?id=2625 (CVE-2016-10095)
http://bugzilla.maptools.org/show_bug.cgi?id=2564 (CVE-2015-7554)
http://bugzilla.maptools.org/show_bug.cgi?id=2561 (CVE-2016-5318)
http://bugzilla.maptools.org/show_bug.cgi?id=2499 (CVE-2014-8128)
http://bugzilla.maptools.org/show_bug.cgi?id=2441
http://bugzilla.maptools.org/show_bug.cgi?id=2433
diff --git a/ChangeLog b/ChangeLog
index 04881ba7..ebd1a3c0 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,23 @@
+2017-06-01 Even Rouault <even.rouault at spatialys.com>
+
+ * libtiff/tif_dirinfo.c, tif_dirread.c: add _TIFFCheckFieldIsValidForCodec(),
+ and use it in TIFFReadDirectory() so as to ignore fields whose tag is a
+ codec-specified tag but this codec is not enabled. This avoids TIFFGetField()
+ to behave differently depending on whether the codec is enabled or not, and
+ thus can avoid stack based buffer overflows in a number of TIFF utilities
+ such as tiffsplit, tiffcmp, thumbnail, etc.
+ Patch derived from 0063-Handle-properly-CODEC-specific-tags.patch
+ (http://bugzilla.maptools.org/show_bug.cgi?id=2580) by Raphaël Hertzog.
+ Fixes:
+ http://bugzilla.maptools.org/show_bug.cgi?id=2580
+ http://bugzilla.maptools.org/show_bug.cgi?id=2693
+ http://bugzilla.maptools.org/show_bug.cgi?id=2625 (CVE-2016-10095)
+ http://bugzilla.maptools.org/show_bug.cgi?id=2564 (CVE-2015-7554)
+ http://bugzilla.maptools.org/show_bug.cgi?id=2561 (CVE-2016-5318)
+ http://bugzilla.maptools.org/show_bug.cgi?id=2499 (CVE-2014-8128)
+ http://bugzilla.maptools.org/show_bug.cgi?id=2441
+ http://bugzilla.maptools.org/show_bug.cgi?id=2433
+
2017-05-29 Even Rouault <even.rouault at spatialys.com>
* libtiff/tif_getimage.c: initYCbCrConversion(): stricter validation for
diff --git a/libtiff/tif_dir.h b/libtiff/tif_dir.h
index 6af5f3dc..5a380767 100644
--- a/libtiff/tif_dir.h
+++ b/libtiff/tif_dir.h
@@ -1,4 +1,4 @@
-/* $Id: tif_dir.h,v 1.54 2011-02-18 20:53:05 fwarmerdam Exp $ */
+/* $Id: tif_dir.h,v 1.55 2017-06-01 12:44:04 erouault Exp $ */
/*
* Copyright (c) 1988-1997 Sam Leffler
@@ -291,6 +291,7 @@ struct _TIFFField {
extern int _TIFFMergeFields(TIFF*, const TIFFField[], uint32);
extern const TIFFField* _TIFFFindOrRegisterField(TIFF *, uint32, TIFFDataType);
extern TIFFField* _TIFFCreateAnonField(TIFF *, uint32, TIFFDataType);
+extern int _TIFFCheckFieldIsValidForCodec(TIFF *tif, ttag_t tag);
#if defined(__cplusplus)
}
diff --git a/libtiff/tif_dirinfo.c b/libtiff/tif_dirinfo.c
index 23ad0020..4904f540 100644
--- a/libtiff/tif_dirinfo.c
+++ b/libtiff/tif_dirinfo.c
@@ -1,4 +1,4 @@
-/* $Id: tif_dirinfo.c,v 1.126 2016-11-18 02:52:13 bfriesen Exp $ */
+/* $Id: tif_dirinfo.c,v 1.127 2017-06-01 12:44:04 erouault Exp $ */
/*
* Copyright (c) 1988-1997 Sam Leffler
@@ -956,6 +956,109 @@ TIFFMergeFieldInfo(TIFF* tif, const TIFFFieldInfo info[], uint32 n)
return 0;
}
+int
+_TIFFCheckFieldIsValidForCodec(TIFF *tif, ttag_t tag)
+{
+ /* Filter out non-codec specific tags */
+ switch (tag) {
+ /* Shared tags */
+ case TIFFTAG_PREDICTOR:
+ /* JPEG tags */
+ case TIFFTAG_JPEGTABLES:
+ /* OJPEG tags */
+ case TIFFTAG_JPEGIFOFFSET:
+ case TIFFTAG_JPEGIFBYTECOUNT:
+ case TIFFTAG_JPEGQTABLES:
+ case TIFFTAG_JPEGDCTABLES:
+ case TIFFTAG_JPEGACTABLES:
+ case TIFFTAG_JPEGPROC:
+ case TIFFTAG_JPEGRESTARTINTERVAL:
+ /* CCITT* */
+ case TIFFTAG_BADFAXLINES:
+ case TIFFTAG_CLEANFAXDATA:
+ case TIFFTAG_CONSECUTIVEBADFAXLINES:
+ case TIFFTAG_GROUP3OPTIONS:
+ case TIFFTAG_GROUP4OPTIONS:
+ break;
+ default:
+ return 1;
+ }
+ /* Check if codec specific tags are allowed for the current
+ * compression scheme (codec) */
+ switch (tif->tif_dir.td_compression) {
+ case COMPRESSION_LZW:
+ if (tag == TIFFTAG_PREDICTOR)
+ return 1;
+ break;
+ case COMPRESSION_PACKBITS:
+ /* No codec-specific tags */
+ break;
+ case COMPRESSION_THUNDERSCAN:
+ /* No codec-specific tags */
+ break;
+ case COMPRESSION_NEXT:
+ /* No codec-specific tags */
+ break;
+ case COMPRESSION_JPEG:
+ if (tag == TIFFTAG_JPEGTABLES)
+ return 1;
+ break;
+ case COMPRESSION_OJPEG:
+ switch (tag) {
+ case TIFFTAG_JPEGIFOFFSET:
+ case TIFFTAG_JPEGIFBYTECOUNT:
+ case TIFFTAG_JPEGQTABLES:
+ case TIFFTAG_JPEGDCTABLES:
+ case TIFFTAG_JPEGACTABLES:
+ case TIFFTAG_JPEGPROC:
+ case TIFFTAG_JPEGRESTARTINTERVAL:
+ return 1;
+ }
+ break;
+ case COMPRESSION_CCITTRLE:
+ case COMPRESSION_CCITTRLEW:
+ case COMPRESSION_CCITTFAX3:
+ case COMPRESSION_CCITTFAX4:
+ switch (tag) {
+ case TIFFTAG_BADFAXLINES:
+ case TIFFTAG_CLEANFAXDATA:
+ case TIFFTAG_CONSECUTIVEBADFAXLINES:
+ return 1;
+ case TIFFTAG_GROUP3OPTIONS:
+ if (tif->tif_dir.td_compression == COMPRESSION_CCITTFAX3)
+ return 1;
+ break;
+ case TIFFTAG_GROUP4OPTIONS:
+ if (tif->tif_dir.td_compression == COMPRESSION_CCITTFAX4)
+ return 1;
+ break;
+ }
+ break;
+ case COMPRESSION_JBIG:
+ /* No codec-specific tags */
+ break;
+ case COMPRESSION_DEFLATE:
+ case COMPRESSION_ADOBE_DEFLATE:
+ if (tag == TIFFTAG_PREDICTOR)
+ return 1;
+ break;
+ case COMPRESSION_PIXARLOG:
+ if (tag == TIFFTAG_PREDICTOR)
+ return 1;
+ break;
+ case COMPRESSION_SGILOG:
+ case COMPRESSION_SGILOG24:
+ /* No codec-specific tags */
+ break;
+ case COMPRESSION_LZMA:
+ if (tag == TIFFTAG_PREDICTOR)
+ return 1;
+ break;
+
+ }
+ return 0;
+}
+
/* vim: set ts=8 sts=8 sw=8 noet: */
/*
diff --git a/libtiff/tif_dirread.c b/libtiff/tif_dirread.c
index 772ebaf7..acde78b5 100644
--- a/libtiff/tif_dirread.c
+++ b/libtiff/tif_dirread.c
@@ -1,4 +1,4 @@
-/* $Id: tif_dirread.c,v 1.208 2017-04-27 15:46:22 erouault Exp $ */
+/* $Id: tif_dirread.c,v 1.209 2017-06-01 12:44:04 erouault Exp $ */
/*
* Copyright (c) 1988-1997 Sam Leffler
@@ -3580,6 +3580,10 @@ TIFFReadDirectory(TIFF* tif)
goto bad;
dp->tdir_tag=IGNORE;
break;
+ default:
+ if( !_TIFFCheckFieldIsValidForCodec(tif, dp->tdir_tag) )
+ dp->tdir_tag=IGNORE;
+ break;
}
}
}
From fe8d7165956b88df4837034a9161dc5fd20cf67a Mon Sep 17 00:00:00 2001
From: Even Rouault <even.rouault@spatialys.com>
Date: Mon, 26 Jun 2017 15:19:59 +0000
Subject: [PATCH] * libtiff/tif_jbig.c: fix memory leak in error code path of
JBIGDecode() Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2706 Reported
by team OWL337
* libtiff/tif_jpeg.c: error out at decoding time if anticipated libjpeg
---
ChangeLog | 8 +++++++-
libtiff/tif_jbig.c | 1 +
2 files changed, 8 insertions(+), 1 deletion(-)
diff --git a/ChangeLog b/ChangeLog
index bc5096e7..ecd70534 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+2017-06-26 Even Rouault <even.rouault at spatialys.com>
+
+ * libtiff/tif_jbig.c: fix memory leak in error code path of JBIGDecode()
+ Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2706
+ Reported by team OWL337
+
2017-06-01 Even Rouault <even.rouault at spatialys.com>
* libtiff/tif_dirinfo.c, tif_dirread.c: add _TIFFCheckFieldIsValidForCodec(),
diff --git a/libtiff/tif_jbig.c b/libtiff/tif_jbig.c
index 5f5f75e2..c75f31d9 100644
--- a/libtiff/tif_jbig.c
+++ b/libtiff/tif_jbig.c
@@ -94,6 +94,7 @@ static int JBIGDecode(TIFF* tif, uint8* buffer, tmsize_t size, uint16 s)
jbg_strerror(decodeStatus)
#endif
);
+ jbg_dec_free(&decoder);
return 0;
}
From 1077fad562e03d1cad591dd10163dd80ad63ab0e Mon Sep 17 00:00:00 2001
From: Even Rouault <even.rouault@spatialys.com>
Date: Fri, 30 Jun 2017 13:11:18 +0000
Subject: [PATCH] * libtiff/tif_read.c, tiffiop.h: add a
_TIFFReadEncodedStripAndAllocBuffer() function, variant of
TIFFReadEncodedStrip() that allocates the decoded buffer only after a first
successful TIFFFillStrip(). This avoids excessive memory allocation on
corrupted files. * libtiff/tif_getimage.c: use
_TIFFReadEncodedStripAndAllocBuffer(). Fixes
http://bugzilla.maptools.org/show_bug.cgi?id=2708 and
https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=2433 . Credit to OSS
Fuzz
---
ChangeLog | 11 +++++++
libtiff/tif_getimage.c | 59 ++++++++++++++++++++++----------------
libtiff/tif_read.c | 78 +++++++++++++++++++++++++++++++++++++++++++-------
libtiff/tiffiop.h | 5 ++++
4 files changed, 118 insertions(+), 35 deletions(-)
diff --git a/ChangeLog b/ChangeLog
index c969f9e2..6f085e09 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,14 @@
+2017-06-30 Even Rouault <even.rouault at spatialys.com>
+
+ * libtiff/tif_read.c, tiffiop.h: add a _TIFFReadEncodedStripAndAllocBuffer()
+ function, variant of TIFFReadEncodedStrip() that allocates the
+ decoded buffer only after a first successful TIFFFillStrip(). This avoids
+ excessive memory allocation on corrupted files.
+ * libtiff/tif_getimage.c: use _TIFFReadEncodedStripAndAllocBuffer().
+ Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2708 and
+ https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=2433 .
+ Credit to OSS Fuzz
+
2017-06-26 Even Rouault <even.rouault at spatialys.com>
* libtiff/tif_jbig.c: fix memory leak in error code path of JBIGDecode()
diff --git a/libtiff/tif_getimage.c b/libtiff/tif_getimage.c
index cee8e930..cc6e8f30 100644
--- a/libtiff/tif_getimage.c
+++ b/libtiff/tif_getimage.c
@@ -905,26 +905,22 @@ gtStripContig(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
tileContigRoutine put = img->put.contig;
uint32 row, y, nrow, nrowsub, rowstoread;
tmsize_t pos;
- unsigned char* buf;
+ unsigned char* buf = NULL;
uint32 rowsperstrip;
uint16 subsamplinghor,subsamplingver;
uint32 imagewidth = img->width;
tmsize_t scanline;
int32 fromskew, toskew;
int ret = 1, flip;
+ tmsize_t maxstripsize;
TIFFGetFieldDefaulted(tif, TIFFTAG_YCBCRSUBSAMPLING, &subsamplinghor, &subsamplingver);
if( subsamplingver == 0 ) {
TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif), "Invalid vertical YCbCr subsampling");
return (0);
}
-
- buf = (unsigned char*) _TIFFmalloc(TIFFStripSize(tif));
- if (buf == 0) {
- TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif), "No space for strip buffer");
- return (0);
- }
- _TIFFmemset(buf, 0, TIFFStripSize(tif));
+
+ maxstripsize = TIFFStripSize(tif);
flip = setorientation(img);
if (flip & FLIP_VERTICALLY) {
@@ -946,11 +942,12 @@ gtStripContig(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
nrowsub = nrow;
if ((nrowsub%subsamplingver)!=0)
nrowsub+=subsamplingver-nrowsub%subsamplingver;
- if (TIFFReadEncodedStrip(tif,
+ if (_TIFFReadEncodedStripAndAllocBuffer(tif,
TIFFComputeStrip(tif,row+img->row_offset, 0),
- buf,
+ (void**)(&buf),
+ maxstripsize,
((row + img->row_offset)%rowsperstrip + nrowsub) * scanline)==(tmsize_t)(-1)
- && img->stoponerr)
+ && (buf == NULL || img->stoponerr))
{
ret = 0;
break;
@@ -994,8 +991,8 @@ gtStripSeparate(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
{
TIFF* tif = img->tif;
tileSeparateRoutine put = img->put.separate;
- unsigned char *buf;
- unsigned char *p0, *p1, *p2, *pa;
+ unsigned char *buf = NULL;
+ unsigned char *p0 = NULL, *p1 = NULL, *p2 = NULL, *pa = NULL;
uint32 row, y, nrow, rowstoread;
tmsize_t pos;
tmsize_t scanline;
@@ -1014,15 +1011,6 @@ gtStripSeparate(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif), "Integer overflow in %s", "gtStripSeparate");
return (0);
}
- p0 = buf = (unsigned char *)_TIFFmalloc(bufsize);
- if (buf == 0) {
- TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif), "No space for tile buffer");
- return (0);
- }
- _TIFFmemset(buf, 0, bufsize);
- p1 = p0 + stripsize;
- p2 = p1 + stripsize;
- pa = (alpha?(p2+stripsize):NULL);
flip = setorientation(img);
if (flip & FLIP_VERTICALLY) {
@@ -1040,7 +1028,6 @@ gtStripSeparate(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
case PHOTOMETRIC_MINISBLACK:
case PHOTOMETRIC_PALETTE:
colorchannels = 1;
- p2 = p1 = p0;
break;
default:
@@ -1056,7 +1043,31 @@ gtStripSeparate(TIFFRGBAImage* img, uint32* raster, uint32 w, uint32 h)
rowstoread = rowsperstrip - (row + img->row_offset) % rowsperstrip;
nrow = (row + rowstoread > h ? h - row : rowstoread);
offset_row = row + img->row_offset;
- if (TIFFReadEncodedStrip(tif, TIFFComputeStrip(tif, offset_row, 0),
+ if( buf == NULL )
+ {
+ if (_TIFFReadEncodedStripAndAllocBuffer(
+ tif, TIFFComputeStrip(tif, offset_row, 0),
+ (void**) &buf, bufsize,
+ ((row + img->row_offset)%rowsperstrip + nrow) * scanline)==(tmsize_t)(-1)
+ && (buf == NULL || img->stoponerr))
+ {
+ ret = 0;
+ break;
+ }
+ p0 = buf;
+ if( colorchannels == 1 )
+ {
+ p2 = p1 = p0;
+ pa = (alpha?(p0+3*stripsize):NULL);
+ }
+ else
+ {
+ p1 = p0 + stripsize;
+ p2 = p1 + stripsize;
+ pa = (alpha?(p2+stripsize):NULL);
+ }
+ }
+ else if (TIFFReadEncodedStrip(tif, TIFFComputeStrip(tif, offset_row, 0),
p0, ((row + img->row_offset)%rowsperstrip + nrow) * scanline)==(tmsize_t)(-1)
&& img->stoponerr)
{
diff --git a/libtiff/tif_read.c b/libtiff/tif_read.c
index fc0072e7..047305ab 100644
--- a/libtiff/tif_read.c
+++ b/libtiff/tif_read.c
@@ -442,18 +442,17 @@ TIFFReadScanline(TIFF* tif, void* buf, uint32 row, uint16 sample)
}
/*
- * Read a strip of data and decompress the specified
- * amount into the user-supplied buffer.
+ * Calculate the strip size according to the number of
+ * rows in the strip (check for truncated last strip on any
+ * of the separations).
*/
-tmsize_t
-TIFFReadEncodedStrip(TIFF* tif, uint32 strip, void* buf, tmsize_t size)
+static tmsize_t TIFFReadEncodedStripGetStripSize(TIFF* tif, uint32 strip, uint16* pplane)
{
static const char module[] = "TIFFReadEncodedStrip";
TIFFDirectory *td = &tif->tif_dir;
uint32 rowsperstrip;
uint32 stripsperplane;
uint32 stripinplane;
- uint16 plane;
uint32 rows;
tmsize_t stripsize;
if (!TIFFCheckRead(tif,0))
@@ -465,23 +464,37 @@ TIFFReadEncodedStrip(TIFF* tif, uint32 strip, void* buf, tmsize_t size)
(unsigned long)td->td_nstrips);
return((tmsize_t)(-1));
}
- /*
- * Calculate the strip size according to the number of
- * rows in the strip (check for truncated last strip on any
- * of the separations).
- */
+
rowsperstrip=td->td_rowsperstrip;
if (rowsperstrip>td->td_imagelength)
rowsperstrip=td->td_imagelength;
stripsperplane= TIFFhowmany_32_maxuint_compat(td->td_imagelength, rowsperstrip);
stripinplane=(strip%stripsperplane);
- plane=(uint16)(strip/stripsperplane);
+ if( pplane ) *pplane=(uint16)(strip/stripsperplane);
rows=td->td_imagelength-stripinplane*rowsperstrip;
if (rows>rowsperstrip)
rows=rowsperstrip;
stripsize=TIFFVStripSize(tif,rows);
if (stripsize==0)
return((tmsize_t)(-1));
+ return stripsize;
+}
+
+/*
+ * Read a strip of data and decompress the specified
+ * amount into the user-supplied buffer.
+ */
+tmsize_t
+TIFFReadEncodedStrip(TIFF* tif, uint32 strip, void* buf, tmsize_t size)
+{
+ static const char module[] = "TIFFReadEncodedStrip";
+ TIFFDirectory *td = &tif->tif_dir;
+ tmsize_t stripsize;
+ uint16 plane;
+
+ stripsize=TIFFReadEncodedStripGetStripSize(tif, strip, &plane);
+ if (stripsize==((tmsize_t)(-1)))
+ return((tmsize_t)(-1));
/* shortcut to avoid an extra memcpy() */
if( td->td_compression == COMPRESSION_NONE &&
@@ -510,6 +523,49 @@ TIFFReadEncodedStrip(TIFF* tif, uint32 strip, void* buf, tmsize_t size)
return(stripsize);
}
+/* Variant of TIFFReadEncodedStrip() that does
+ * * if *buf == NULL, *buf = _TIFFmalloc(bufsizetoalloc) only after TIFFFillStrip() has
+ * suceeded. This avoid excessive memory allocation in case of truncated
+ * file.
+ * * calls regular TIFFReadEncodedStrip() if *buf != NULL
+ */
+tmsize_t
+_TIFFReadEncodedStripAndAllocBuffer(TIFF* tif, uint32 strip,
+ void **buf, tmsize_t bufsizetoalloc,
+ tmsize_t size_to_read)
+{
+ tmsize_t this_stripsize;
+ uint16 plane;
+
+ if( *buf != NULL )
+ {
+ return TIFFReadEncodedStrip(tif, strip, *buf, size_to_read);
+ }
+
+ this_stripsize=TIFFReadEncodedStripGetStripSize(tif, strip, &plane);
+ if (this_stripsize==((tmsize_t)(-1)))
+ return((tmsize_t)(-1));
+
+ if ((size_to_read!=(tmsize_t)(-1))&&(size_to_read<this_stripsize))
+ this_stripsize=size_to_read;
+ if (!TIFFFillStrip(tif,strip))
+ return((tmsize_t)(-1));
+
+ *buf = _TIFFmalloc(bufsizetoalloc);
+ if (*buf == NULL) {
+ TIFFErrorExt(tif->tif_clientdata, TIFFFileName(tif), "No space for strip buffer");
+ return((tmsize_t)(-1));
+ }
+ _TIFFmemset(*buf, 0, bufsizetoalloc);
+
+ if ((*tif->tif_decodestrip)(tif,*buf,this_stripsize,plane)<=0)
+ return((tmsize_t)(-1));
+ (*tif->tif_postdecode)(tif,*buf,this_stripsize);
+ return(this_stripsize);
+
+
+}
+
static tmsize_t
TIFFReadRawStrip1(TIFF* tif, uint32 strip, void* buf, tmsize_t size,
const char* module)
diff --git a/libtiff/tiffiop.h b/libtiff/tiffiop.h
index 846ade03..7f0b90f7 100644
--- a/libtiff/tiffiop.h
+++ b/libtiff/tiffiop.h
@@ -365,6 +365,11 @@ extern void* _TIFFCheckRealloc(TIFF*, void*, tmsize_t, tmsize_t, const char*);
extern double _TIFFUInt64ToDouble(uint64);
extern float _TIFFUInt64ToFloat(uint64);
+extern tmsize_t
+_TIFFReadEncodedStripAndAllocBuffer(TIFF* tif, uint32 strip,
+ void **buf, tmsize_t bufsizetoalloc,
+ tmsize_t size_to_read);
+
extern int TIFFInitDumpMode(TIFF*, int);
#ifdef PACKBITS_SUPPORT
extern int TIFFInitPackBits(TIFF*, int);
From 6173a57d39e04d68b139f8c1aa499a24dbe74ba1 Mon Sep 17 00:00:00 2001
From: Even Rouault <even.rouault@spatialys.com>
Date: Fri, 30 Jun 2017 17:29:44 +0000
Subject: [PATCH] * libtiff/tif_dirwrite.c: in
TIFFWriteDirectoryTagCheckedXXXX() functions associated with LONG8/SLONG8
data type, replace assertion that the file is BigTIFF, by a non-fatal error.
Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2712 Reported by team
OWL337
---
ChangeLog | 8 ++++++++
libtiff/tif_dirwrite.c | 20 ++++++++++++++++----
2 files changed, 24 insertions(+), 4 deletions(-)
diff --git a/ChangeLog b/ChangeLog
index 6f085e09..77a64385 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,13 @@
2017-06-30 Even Rouault <even.rouault at spatialys.com>
+ * libtiff/tif_dirwrite.c: in TIFFWriteDirectoryTagCheckedXXXX()
+ functions associated with LONG8/SLONG8 data type, replace assertion that
+ the file is BigTIFF, by a non-fatal error.
+ Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2712
+ Reported by team OWL337
+
+2017-06-30 Even Rouault <even.rouault at spatialys.com>
+
* libtiff/tif_read.c, tiffiop.h: add a _TIFFReadEncodedStripAndAllocBuffer()
function, variant of TIFFReadEncodedStrip() that allocates the
decoded buffer only after a first successful TIFFFillStrip(). This avoids
diff --git a/libtiff/tif_dirwrite.c b/libtiff/tif_dirwrite.c
index 2967da58..8d6686ba 100644
--- a/libtiff/tif_dirwrite.c
+++ b/libtiff/tif_dirwrite.c
@@ -2111,7 +2111,10 @@ TIFFWriteDirectoryTagCheckedLong8(TIFF* tif, uint32* ndir, TIFFDirEntry* dir, ui
{
uint64 m;
assert(sizeof(uint64)==8);
- assert(tif->tif_flags&TIFF_BIGTIFF);
+ if( !(tif->tif_flags&TIFF_BIGTIFF) ) {
+ TIFFErrorExt(tif->tif_clientdata,"TIFFWriteDirectoryTagCheckedLong8","LONG8 not allowed for ClassicTIFF");
+ return(0);
+ }
m=value;
if (tif->tif_flags&TIFF_SWAB)
TIFFSwabLong8(&m);
@@ -2124,7 +2127,10 @@ TIFFWriteDirectoryTagCheckedLong8Array(TIFF* tif, uint32* ndir, TIFFDirEntry* di
{
assert(count<0x20000000);
assert(sizeof(uint64)==8);
- assert(tif->tif_flags&TIFF_BIGTIFF);
+ if( !(tif->tif_flags&TIFF_BIGTIFF) ) {
+ TIFFErrorExt(tif->tif_clientdata,"TIFFWriteDirectoryTagCheckedLong8","LONG8 not allowed for ClassicTIFF");
+ return(0);
+ }
if (tif->tif_flags&TIFF_SWAB)
TIFFSwabArrayOfLong8(value,count);
return(TIFFWriteDirectoryTagData(tif,ndir,dir,tag,TIFF_LONG8,count,count*8,value));
@@ -2136,7 +2142,10 @@ TIFFWriteDirectoryTagCheckedSlong8(TIFF* tif, uint32* ndir, TIFFDirEntry* dir, u
{
int64 m;
assert(sizeof(int64)==8);
- assert(tif->tif_flags&TIFF_BIGTIFF);
+ if( !(tif->tif_flags&TIFF_BIGTIFF) ) {
+ TIFFErrorExt(tif->tif_clientdata,"TIFFWriteDirectoryTagCheckedLong8","SLONG8 not allowed for ClassicTIFF");
+ return(0);
+ }
m=value;
if (tif->tif_flags&TIFF_SWAB)
TIFFSwabLong8((uint64*)(&m));
@@ -2149,7 +2158,10 @@ TIFFWriteDirectoryTagCheckedSlong8Array(TIFF* tif, uint32* ndir, TIFFDirEntry* d
{
assert(count<0x20000000);
assert(sizeof(int64)==8);
- assert(tif->tif_flags&TIFF_BIGTIFF);
+ if( !(tif->tif_flags&TIFF_BIGTIFF) ) {
+ TIFFErrorExt(tif->tif_clientdata,"TIFFWriteDirectoryTagCheckedLong8","SLONG8 not allowed for ClassicTIFF");
+ return(0);
+ }
if (tif->tif_flags&TIFF_SWAB)
TIFFSwabArrayOfLong8((uint64*)value,count);
return(TIFFWriteDirectoryTagData(tif,ndir,dir,tag,TIFF_SLONG8,count,count*8,value));
component/proxysql/buildout.cfg 0 → 100644
View file @ ce787d53
[buildout]
extends =
../autoconf/buildout.cfg
../automake/buildout.cfg
../libtool/buildout.cfg
../cmake/buildout.cfg
../openssl/buildout.cfg
../patch/buildout.cfg
../git/buildout.cfg
../openssl/buildout.cfg
../bzip2/buildout.cfg
../perl/buildout.cfg
../gnutls/buildout.cfg
../curl/buildout.cfg
../gnutls/buildout.cfg
../libzip/buildout.cfg
../m4/buildout.cfg
../pcre/buildout.cfg
../jemalloc/buildout.cfg
../libmicrohttpd/buildout.cfg
parts =
proxysql
[proxysql]
recipe = slapos.recipe.cmmi
version = v2.0.12
url = https://github.com/sysown/proxysql/archive/${:version}.tar.gz
md5sum = 70ec17fe73703a25730fdd44b6bc3ef5
prefix = @@LOCATION@@
# Patch installation path for SlapOS
pre-configure =
mkdir -p ${:prefix}/bin ${:prefix}/etc/init.d ${:prefix}/lib/systemd/system
sed -ri "
s#(\s)/usr/bin#\1${:prefix}/bin#g
s#(\s)/etc#\1${:prefix}/etc#g
s#(\s)/usr/lib#\1${:prefix}/lib#
s#(\s)/var/lib#\1${:prefix}/lib#g
s#(\s)useradd#\1echo useradd#g
s#(\s)systemctl#\1echo systemctl#g
s#(\s)chkconfig#\1echo chkconfig#g
s#(\s)update-rc.d#\1echo update-rc.d#g" Makefile
configure-command = true
environment =
GIT_VERSION=${:version}
PKG_CONFIG_PATH=${openssl:location}/lib/pkgconfig:${gnutls:location}/lib/pkgconfig:${libgcrypt:location}/lib/pkgconfig:${zlib:location}/lib/pkgconfig:${pcre:location}/lib/pkgconfig
PATH=${m4:location}/bin:${libtool:location}/bin:${libgcrypt:location}/bin:${curl:location}/bin:${perl:location}/bin:${pkgconfig:location}/bin:${bzip2:location}/bin:${autoconf:location}/bin:${git:location}/bin:${automake:location}/bin:${patch:location}/bin:${cmake:location}/bin:%(PATH)s
CXXFLAGS=-I${openssl:location}/include -I${gnutls:location}/include -I${zlib:location}/include
CFLAGS=-I${gnutls:location}/include
LDFLAGS=-L${openssl:location}/lib -Wl,-rpath -Wl,${gnutls:location}/lib -L${gnutls:location}/lib -Wl,-rpath=${curl:location}/lib -L${libtool:location}/lib -L${zlib:location}/lib -Wl,-rpath -Wl,${zlib:location}/lib -L${curl:location}/lib -L${pcre:location}/lib -L${jemalloc:location}/lib -L${libmicrohttpd:location}/lib
CMAKE_INCLUDE_PATH=${openssl:location}/include:${gnutls:location}/include:${curl:location}/include:${pcre:location}/include:${jemalloc:location}/include:${libmicrohttpd:location}/include
CMAKE_LIBRARY_PATH=${openssl:location}/lib:${gnutls:location}/lib:${curl:location}/lib:${pcre:location}/lib:${jemalloc:location}/lib:${libmicrohttpd:location}/lib
LIBTOOL=libtool
ACLOCAL_PATH=${pkgconfig:location}/share/aclocal:${libtool:location}/share/aclocal
component/python-manuel/buildout.cfg 0 → 100644
View file @ ce787d53
[buildout]
extends = ../patch/buildout.cfg
parts = manuel
[manuel]
recipe = zc.recipe.egg
egg = manuel
find-links = https://github.com/benji-york/manuel/tarball/${:revision}/manuel-${versions:manuel}.tar.gz
revision = aeea5ce7e853b8dabb815efb83141c6ddeb62904
[versions]
manuel = 1.10.2.dev0
component/replication-manager/buildout.cfg 0 → 100644
View file @ ce787d53
[buildout]
extends =
../../component/golang/buildout.cfg
parts =
gowork
[gowork]
golang = ${golang1.13:location}
install =
buildflags = -v --tags server --ldflags "-extldflags 'static' -w -s -X main.GoOS=linux -X main.GoArch=amd64 -X main.Version=2.1 -X main.FullVersion=$FULLVERSION -X main.Build=$(date +%FT%T%z) -X main.WithProvisioning=ON -X main.WithOpenSVC=OFF -X main.WithHaproxy=ON -X main.WithMaxscale=ON -X main.WithMariadbshardproxy=ON -X main.WithProxysql=ON -X main.WithSphinx=ON -X main.WithArbitration=OFF -X main.WithArbitrationClient=ON -X main.WithMonitoring=ON -X main.WithHttp=ON -X main.WithBackup=ON -X main.WithMail=ON -X main.WithEnforce=ON -X main.WithDeprecate=ON"
[gowork.goinstall]
depends_gitfetch =
${git.signal18.io_signal18_repman:recipe}
command = set -e
. ${gowork:env.sh}
cd ${git.signal18.io_signal18_repman:location}
export GO111MODULE=on
export FULLVERSION=$(git describe --tags)
go build ${gowork:buildflags} -o ${gowork:bin}/replication-manager
chmod -R u+w .
# Remove binary files provided with replication manager else testnode will complain with 'libXXX => not found'.
rm -f ${git.signal18.io_signal18_repman:location}/share/amd64/darwin/*
rm -f ${git.signal18.io_signal18_repman:location}/share/amd64/linux/*
[git.signal18.io_signal18_repman]
<= go-git-package
go.importpath = github.com/signal18/replication-manager
repository = https://github.com/signal18/replication-manager
branch = 2.1
revision = 9167a82c81af8f7be41cf51bc9be8a37dc3d8c03
\ No newline at end of file
component/restic/buildout.cfg 0 → 100644
View file @ ce787d53
[buildout]
extends =
../../component/golang/buildout.cfg
parts =
restic
[gowork]
golang = ${golang1.13:location}
[restic]
recipe = plone.recipe.command
update-command = ${:command}
stop-on-error = True
# GO111MODULE=on enables go modules support
# the chmod is needed as modules are fetched with u-w
command =
. ${gowork:env.sh} &&
cd ${git.github.com_restic_restic:location} &&
export GO111MODULE=on &&
go run build.go -o ${:output} &&
chmod -R u+w .
output = ${gowork:bin}/restic
location = ${:output}
[git.github.com_restic_restic]
<= go-git-package
go.importpath = github.com/restic/restic
repository = https://github.com/restic/restic
revision = v0.9.6
\ No newline at end of file
component/rina-tools/buildout.cfg
View file @ ce787d53
[buildout] [buildout]
extends = extends =
../defaults.cfg
../automake/buildout.cfg ../automake/buildout.cfg
../git/buildout.cfg ../git/buildout.cfg
../libtool/buildout.cfg ../libtool/buildout.cfg
...@@ -7,6 +8,9 @@ extends = ...@@ -7,6 +8,9 @@ extends =
parts = rina-tools parts = rina-tools
[gcc]
min_version = 0
[irati-stack] [irati-stack]
recipe = slapos.recipe.build:gitclone recipe = slapos.recipe.build:gitclone
repository = https://github.com/jmuchemb/irati-stack.git repository = https://github.com/jmuchemb/irati-stack.git
......
component/socat/buildout.cfg
View file @ ce787d53
[buildout] [buildout]
extends =
../openssl/buildout.cfg
parts = parts =
socat socat
...@@ -7,3 +10,5 @@ recipe = slapos.recipe.cmmi ...@@ -7,3 +10,5 @@ recipe = slapos.recipe.cmmi
url = http://www.dest-unreach.org/socat/download/socat-${:version}.tar.gz url = http://www.dest-unreach.org/socat/download/socat-${:version}.tar.gz
version = 1.7.3.2 version = 1.7.3.2
md5sum = aec3154f7854580cfab0c2d81e910519 md5sum = aec3154f7854580cfab0c2d81e910519
environment =
LDFLAGS=-L${openssl:location}/lib -Wl,-rpath=${openssl:location}/lib
\ No newline at end of file
component/sysbench/buildout.cfg 0 → 100644
View file @ ce787d53
[buildout]
extends =
../autoconf/buildout.cfg
../automake/buildout.cfg
../libtool/buildout.cfg
../gettext/buildout.cfg
../m4/buildout.cfg
../mariadb/buildout.cfg
parts =
sysbench
[sysbench]
recipe = slapos.recipe.cmmi
shared = true
url = https://github.com/akopytov/sysbench/archive/1.0.19.tar.gz
md5sum = 2912bfe7238cac7351459019a84e2557
pre-configure =
aclocal -I${pkgconfig:location}/share/aclocal -I${libtool:location}/share/aclocal -I${gettext:location}/share/aclocal
./autogen.sh
configure-options =
--disable-static
--with-mysql-includes=${mariadb:location}/include/mysql
--with-mysql-libs=${mariadb:location}/lib
environment =
PATH=${m4:location}/bin:${autoconf:location}/bin:${automake:location}/bin:${pkgconfig:location}/bin:${libtool:location}/bin:%(PATH)s
CPPFLAGS=-I${gettext:location}/include -I${mariadb:location}/include
LDFLAGS=-L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib -L${mariadb:location}/lib -Wl,-rpath=${mariadb:location}/lib
ACLOCAL_PATH=${pkgconfig:location}/share/aclocal:${gettext:location}/share/aclocal:${libtool:location}/share/aclocal
\ No newline at end of file
component/zstd/buildout.cfg
View file @ ce787d53
...@@ -10,8 +10,8 @@ parts = ...@@ -10,8 +10,8 @@ parts =
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
shared = true shared = true
url = https://github.com/facebook/zstd/releases/download/v${:version}/zstd-${:version}.tar.gz url = https://github.com/facebook/zstd/releases/download/v${:version}/zstd-${:version}.tar.gz
version = 1.4.4 version = 1.4.5
md5sum = 487f7ee1562dee7c1c8adf85e2a63df9 md5sum = dd0b53631303b8f972dafa6fd34beb0c
location = @@LOCATION@@ location = @@LOCATION@@
configure-command = : configure-command = :
environment = environment =
......
software/build-rina/software.cfg
View file @ ce787d53
...@@ -8,9 +8,6 @@ parts = ...@@ -8,9 +8,6 @@ parts =
template template
download-cache = ${:directory}/download-cache download-cache = ${:directory}/download-cache
[gcc]
min_version = 0
[template] [template]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
# XXX: "template.cfg" is hardcoded in instanciation recipe # XXX: "template.cfg" is hardcoded in instanciation recipe
......
software/buildout-testing/runTestSuite.in
View file @ ce787d53
...@@ -4,6 +4,7 @@ import argparse, os, re, subprocess, sys ...@@ -4,6 +4,7 @@ import argparse, os, re, subprocess, sys
from time import gmtime, strftime, time from time import gmtime, strftime, time
from erp5.util import taskdistribution from erp5.util import taskdistribution
from erp5.util.testsuite import SubprocessError, TestSuite from erp5.util.testsuite import SubprocessError, TestSuite
from pkg_resources import get_distribution
from zc.buildout.buildout import Buildout from zc.buildout.buildout import Buildout
if str is bytes: if str is bytes:
...@@ -40,17 +41,17 @@ class DummyTestResult: ...@@ -40,17 +41,17 @@ class DummyTestResult:
class BuildoutTestSuite(TestSuite): class BuildoutTestSuite(TestSuite):
RUN_RE = re.compile( RUN_RE = re.compile(
r'Ran (?P<all_tests>\d+) tests with' br'Ran (?P<all_tests>\d+) tests with'
' (?P<failures>\d+) failures,' br' (?P<failures>\d+) failures,'
' (?P<errors>\d+) errors and' br' (?P<errors>\d+) errors and'
' (?P<skips>\d+) skipped in') br' (?P<skips>\d+) skipped in')
def run(self, test): def run(self, test):
start = time() start = time()
try: try:
status_dict = self.spawn(os.path.join('bin', 'zope-testrunner'), status_dict = self.spawn(os.path.join('bin', 'zope-testrunner'),
'--test-path', os.path.join(test_dict[test], 'src')) '--test-path', os.path.join(test_dict[test], 'src'))
except SubprocessError, e: except SubprocessError as e:
status_dict = e.status_dict status_dict = e.status_dict
end = time() end = time()
status_dict.update( status_dict.update(
...@@ -93,35 +94,38 @@ def main(): ...@@ -93,35 +94,38 @@ def main():
else: else:
test_result = DummyTestResult(list(test_dict)) test_result = DummyTestResult(list(test_dict))
fd = os.open('buildout.cfg', os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0666) fd = os.open('buildout.cfg', os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o666)
try: try:
os.write(fd, str2bytes("""\ os.write(fd, str2bytes("""\
[buildout] [buildout]
extends = %s
develop =%s develop =%s
parts = test parts = testrunner
newest = false newest = false
versions = versions
[versions] [versions]
%s %s
zope.exceptions = 4.3
zope.interface = 4.7.2
zope.testing = 4.7
[bootstrap] [testrunner]
recipe = zc.recipe.egg recipe = zc.recipe.egg
eggs = zc.buildout eggs =
${:recipe}
[test] zope.testing
recipe = zc.recipe.egg
eggs +=
zope.testrunner zope.testrunner
scripts = scripts =
zope-testrunner zope-testrunner
""" % (os.path.join(slapos_buildout, 'buildout.cfg'), extra-paths =
''.join('\n ' + x for x in test_dict.itervalues()), %s
'\n'.join(x + ' =' for x in test_dict)))) """ % (''.join('\n ' + x for x in test_dict.values()),
'\n'.join(x + ' =' for x in test_dict),
get_distribution('manuel').location,
)))
finally: finally:
os.close(fd) os.close(fd)
Buildout('buildout.cfg', {}).install(['bootstrap']) Buildout('buildout.cfg', {}).install(None)
subprocess.check_call((os.path.join('bin', 'buildout'),))
test_suite = BuildoutTestSuite(1) test_suite = BuildoutTestSuite(1)
while 1: while 1:
......
software/buildout-testing/software.cfg
View file @ ce787d53
[buildout] [buildout]
extends = extends =
../../component/python-manuel/buildout.cfg
../../stack/slapos.cfg ../../stack/slapos.cfg
parts = parts =
slapos-cookbook slapos-cookbook
...@@ -14,6 +15,7 @@ git-executable = ${git:location}/bin/git ...@@ -14,6 +15,7 @@ git-executable = ${git:location}/bin/git
recipe = zc.recipe.egg recipe = zc.recipe.egg
eggs = erp5.util eggs = erp5.util
zc.buildout zc.buildout
${manuel:egg}
scripts = ${:interpreter} scripts = ${:interpreter}
interpreter = ${:_buildout_section_name_} interpreter = ${:_buildout_section_name_}
......
software/erp5/software-shared.cfg 0 → 100644
View file @ ce787d53
[buildout]
extends = software.cfg
shared-parts = /opt/slapgrid/shared-parts
eggs-directory = /opt/slapgrid/shared-eggs
abi-tag-eggs = true
software/neoppod/software-zodb5.cfg
View file @ ce787d53
...@@ -6,7 +6,7 @@ eggs += mock ...@@ -6,7 +6,7 @@ eggs += mock
ZEO-patches = ZEO-patches =
[versions] [versions]
ZODB = 5.5.1 ZODB = 5.6.0
ZEO = 5.2.0 ZEO = 5.2.0
transaction = 2.4.0 transaction = 2.4.0
......
software/repman/buildout.hash.cfg 0 → 100644
View file @ ce787d53
# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
# The only allowed lines here are (regexes):
# - "^#" comments, copied verbatim
# - "^[" section beginings, copied verbatim
# - lines containing an "=" sign which must fit in the following categorie.
# - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
# Copied verbatim.
# - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
# by the re-generation script.
# Re-generated.
# - other lines are copied verbatim
# Substitution (${...:...}), extension ([buildout] extends = ...) and
# section inheritance (< = ...) are NOT supported (but you should really
# not need these here).
[instance.cfg]
filename = instance.cfg.in
md5sum = b41f521b5f7980c64260ed0e5c494450
[instance-repman.cfg]
_update_hash_filename_ = instance-repman.cfg.jinja2.in
md5sum = 7dbaace0d7db0e26d582ad17f36ac9cd
[config-toml.in]
_update_hash_filename_ = templates/config.toml.in
md5sum = 5cfa75ca5a0048a050c0041dfe541f3d
[config-cluster-toml.in]
_update_hash_filename_ = templates/cluster-config.toml.in
md5sum = d2e79a9435082d9420281b4f59a5d464
[nginx.conf.in]
_update_hash_filename_ = templates/nginx.conf.in
md5sum = 0eeb24c6aa0760f0d33c4cc2828ddf30
[template-mariadb.cfg]
_update_hash_filename_ = instance-mariadb.cfg.jinja2.in
md5sum = 189ccee60d0fb53e29431a45e0816bc1
[template-my-cnf]
_update_hash_filename_ = templates/my.cnf.in
md5sum = f3661b788099bb31d71ba6e7d36836d9
[template-mariadb-initial-setup]
_update_hash_filename_ = templates/mariadb_initial_setup.sql.in
md5sum = 9be53e2e92333b93e92556b8a01d9c42
[mariadb-init-root-sql]
_update_hash_filename_ = templates/mariadb_init_root.sql.in
md5sum = d927b5d36410bb02717d5ca125525785
[init-root-wrapper-in]
_update_hash_filename_ = templates/init_root_wrapper.in
md5sum = 83ef59b5afaf4454d368823c33aef9cb
[repman-manager-sh.in]
_update_hash_filename_ = templates/repman-manager.sh.in
md5sum = 50503bec392e31126328f51eadc11634
[dbjobs-in]
_update_hash_filename_ = templates/dbjobs.in
md5sum = d2ebd2ec55bf8489789a52c808729925
[mysqld-need-start.sh.in]
_update_hash_filename_ = templates/mysqld-need-start.sh.in
md5sum = e9bcee5dc1318fe3acda2663472214f5
[proxy-need-start-stop.sh.in]
_update_hash_filename_ = templates/proxy-need-start-stop.sh.in
md5sum = 455aaf369bf5141758dc57f2c0e67b08
software/repman/instance-mariadb.cfg.jinja2.in 0 → 100644
View file @ ce787d53
{% set part_list = [] -%}
{% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%}
{% set use_ipv6 = slapparameter_dict.get('use-ipv6', True) -%}
{% set port = slapparameter_dict['tcp-port'] %}
{% set host = (ipv4_set | list)[0] -%}
{% if use_ipv6 -%}
{% set ip = (ipv6_set | list)[0] -%}
{% set host = '[' ~ ip ~ ']' -%}
{% else -%}
{% set ip = (ipv4_set | list)[0] -%}
{% endif -%}
{% set dash = parameter_dict['dash-location'] ~ '/bin/dash' %}
{% set database_list = slapparameter_dict['database-list'] -%}
# XXX- TODO: add cron with check db need restart then restart
# API check restard needed: http://repman/api/clusters/{clusterName}/servers/{serverName}/{serverPort}/need-restart
[{{ section('publish') }}]
recipe = slapos.cookbook:publish.serialised
-extends = publish-early
database-host = {{ host }}:{{ port }}
monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
partition-path = ${buildout:directory}
receiver-port = ${dbjob-parameter:socat-port}
[publish-early]
recipe = slapos.cookbook:publish-early
-init =
database-list = {{ dumps(database_list) }}
[jinja2-template-base]
recipe = slapos.recipe.template:jinja2
mode = 644
[jinja2-template-executable]
< = jinja2-template-base
mode = 755
[simplefile]
< = jinja2-template-base
template = inline:{{ '{{ content }}' }}
{% macro simplefile(section_name, file_path, content, mode='') -%}
{% set content_section_name = section_name ~ '-content' -%}
[{{ content_section_name }}]
content = {{ dumps(content) }}
[{{ section(section_name) }}]
< = simplefile
rendered = {{ file_path }}
context = key content {{content_section_name}}:content
mode = {{ mode }}
{%- endmacro %}
[my-cnf-parameters]
socket = ${directory:run}/mariadb.sock
ip = {{ ip }}
data-directory = ${directory:srv}/mariadb
pid-file = ${directory:run}/mariadb.pid
plugin-directory = {{ dumps(parameter_dict['mroonga-mariadb-plugin-dir']) }}
groonga-plugins-path = {{ parameter_dict['groonga-plugins-path'] }}
innodb-buffer-pool-size = {{ dumps(slapparameter_dict.get('innodb-buffer-pool-size', 0)) }}
innodb-buffer-pool-instances = {{ dumps(slapparameter_dict.get('innodb-buffer-pool-instances', 0)) }}
innodb-log-file-size = {{ dumps(slapparameter_dict.get('innodb-log-file-size', 0)) }}
innodb-file-per-table = {{ dumps(slapparameter_dict.get('innodb-file-per-table', 0)) }}
innodb-log-buffer-size = {{ dumps(slapparameter_dict.get('innodb-log-buffer-size', 0)) }}
relaxed-writes = {{ dumps(slapparameter_dict.get('relaxed-writes', False)) }}
ssl-crt = ${directory:mariadb-ssl}/crt.pem
ssl-key = ${directory:mariadb-ssl}/key.pem
ssl-ca-crt = ${certificate-authority:ca-dir}/cacert.pem
[my-cnf]
< = jinja2-template-base
rendered = ${directory:etc}/mariadb.cnf
template = {{ parameter_dict['template-my-cnf'] }}
context = section parameter_dict my-cnf-parameters
[init-root-sql]
< = jinja2-template-base
rendered = ${directory:etc}/.init-root.sql
template = {{ parameter_dict['template-mariadb-init-root'] }}
context = section parameter_dict init-script-parameters
mode = 600
[init-script-parameters]
password = {{ slapparameter_dict['root-password'] }}
database-list = {{ dumps(database_list) }}
mroonga-mariadb-install-sql = {{ dumps(parameter_dict['mroonga-mariadb-install-sql']) }}
root-user = repman
heartbeat-user = {{ slapparameter_dict['heartbeat-user'] }}
[init-script]
< = jinja2-template-executable
# XXX: is there a better location ?
rendered = ${directory:etc}/mariadb_initial_setup.sql
template = {{ parameter_dict['template-mariadb-initial-setup'] }}
context = section parameter_dict init-script-parameters
[update-mysql]
recipe = slapos.cookbook:generic.mysql.wrap_update_mysql
output = ${directory:bin}/mariadb_update
binary = ${binary-wrap-mysql_upgrade:wrapper-path}
mysql = ${binary-wrap-mysql:wrapper-path}
init-script = ${init-script:rendered}
mysql_tzinfo_to_sql = ${binary-wrap-mysql_tzinfo_to_sql:wrapper-path}
[{{ section('update-mysql-script') }}]
< = jinja2-template-executable
rendered = ${directory:scripts}/mariadb_update
init-password = ${directory:etc}/.init-passwd.done
upgrade-done = ${directory:lib}/mariadb-update-done
context =
key init_password_done :init-password
key upgrade_done :upgrade-done
key init_root_sql init-root-sql:rendered
key mysql_update update-mysql:output
raw mysql_conf ${directory:etc}/mysql/my.cnf
raw dash_bin {{ dash }}
raw mysql_bin {{ parameter_dict['mariadb-location'] }}/bin/mysql
template = {{ parameter_dict['template-init-root-wrapper'] }}
[mysqld]
< = jinja2-template-executable
rendered = ${directory:bin}/mysqld
template = {{ parameter_dict['template-mysqld-wrapper'] }}
context =
key defaults_file install-mysql-config:config
key datadir my-cnf-parameters:data-directory
key environ :environ
environ =
GRN_PLUGINS_PATH='${my-cnf-parameters:groonga-plugins-path}'
ODBCSYSINI='${directory:etc}'
LD_LIBRARY_PATH=$${LD_LIBRARY_PATH:+$LD_LIBRARY_PATH:}'{{ parameter_dict['unixodbc-location'] }}/lib'
{%- for variable in slapparameter_dict.get('environment-variables', ()) %}
{{ variable }}
{%- endfor %}
[ca-mysqld]
<= certificate-authority
recipe = slapos.cookbook:certificate_authority.request
key-file = ${my-cnf-parameters:ssl-key}
cert-file = ${my-cnf-parameters:ssl-crt}
executable = ${mysqld:rendered}
wrapper = ${directory:controller}/mariadb
{% import "supervisord_lib" as supervisord_lib with context %}
{{ supervisord_lib.supervisord("mariadb-ctl", buildout_bin_directory, supervisord_conf, use_service_hash=False) }}
{% do part_list.append("supervisord-mariadb-ctl") -%}
{% set maradb_program_dict = {"name": "mariadb", "command": "${ca-mysqld:wrapper}",
"stopwaitsecs": 86400, "environment": [],
"stdout_logfile": "${directory:log}/mariadb_stdout.log",
"stderr_logfile": "${directory:log}/mariadb_stdout.log" } %}
{{ supervisord_lib.supervisord_program("mariadb", maradb_program_dict) }}
{% do part_list.append("supervisord-mariadb") %}
[odbc-ini-text]
text = {{ dumps(slapparameter_dict.get('odbc-ini', '').encode('base64')) }}
[{{ section('odbc-ini') }}]
< = jinja2-template-base
rendered = ${directory:etc}/odbc.ini
template = inline:{% raw -%}
{{ parameter_dict['text'].decode('base64') }}
{%- endraw %}
context = section parameter_dict odbc-ini-text
[{{ section('logrotate-entry-mariadb') }}]
< = logrotate-entry-base
name = mariadb
log = ${dbjob-parameter:log-dir}/errors.log ${dbjob-parameter:log-dir}/sql-errors
post = "${binary-wrap-mysql:wrapper-path}" -B -e "FLUSH LOGS"
[{{ section('binary-link') }}]
recipe = slapos.cookbook:symbolic.link
target-directory = ${directory:bin}
link-binary = {{ dumps(parameter_dict['link-binary']) }}
[binary-wrap-base]
recipe = slapos.cookbook:wrapper
# Note: --defaults-file must be the first argument, otherwise wrapped binary
# will reject it.
command-line =
"{{ parameter_dict['mariadb-location'] }}/bin/${:command}"
--defaults-file="${directory:etc}/mysql/my.cnf" --protocol=socket ${:extra-args}
wrapper-path = ${directory:bin}/${:command}
extra-args =
[binary-wrap-mysql]
<= binary-wrap-base
command = mysql
[binary-wrap-mysqldump]
<= binary-wrap-base
command = mysqldump
[binary-wrap-mysql_upgrade]
<= binary-wrap-base
command = mysql_upgrade
extra-args = --skip-write-binlog
[binary-wrap-mysqladmin]
<= binary-wrap-base
command = mysqladmin
[binary-wrap-mysql_tzinfo_to_sql]
<= binary-wrap-base
command-line = "{{ parameter_dict['mariadb-location'] }}/bin/${:command}" --skip-write-binlog
command = mysql_tzinfo_to_sql
[binary-wrap-pt-digest]
<= binary-wrap-base
command-line = "{{ parameter_dict['percona-tools-location'] }}/bin/${:command}"
command = pt-query-digest
[directory]
recipe = slapos.cookbook:mkdirectory
bin = ${buildout:directory}/bin
etc = ${buildout:directory}/etc
scripts = ${:etc}/run
services = ${:etc}/service
controller = ${:etc}/controller
plugin = ${:etc}/plugin
srv = ${buildout:directory}/srv
tmp = ${buildout:directory}/tmp
backup = ${:srv}/backup
mariadb-backup-full = ${:backup}/mariadb-full
mariadb-backup-incremental = ${:backup}/mariadb-incremental
mariadb-ssl = ${:etc}/mariadb-ssl
var = ${buildout:directory}/var
lib = ${:var}/lib
mysql = ${:lib}/mysql
log = ${:var}/log
run = ${:var}/run
config-tmp = ${:tmp}/config
custom = ${directory:etc}/mysql/custom
[dbjob-parameter]
bash-bin = {{ bash_bin }}
db-user = ${init-script-parameters:root-user}
db-password = ${init-script-parameters:password}
mysql-dir = ${directory:mysql}
dbjob-cnf = ${directory:etc}/mysql/my.cnf
log-dir = ${directory:mysql}/.system/logs
tmp-dir = ${directory:tmp}
mysqld-socket = ${my-cnf-parameters:socket}
socat-port = {{ int(port) + 9 }}
restart-script = ${mysqld-restart-script:rendered}
socat-location = {{ parameter_dict['socat-location'] }}
mysql-location = {{ parameter_dict['mariadb-location'] }}
gzip-location = {{ parameter_dict['gzip-location'] }}
ip = {{ ip }}
host = {{ host }}
port = {{ port }}
use-ipv6 = {{ dumps(use_ipv6) }}
[dbjobs-executable]
< = jinja2-template-executable
rendered = ${directory:bin}/dbjobs
context =
section parameter_dict dbjob-parameter
template = {{ parameter_dict['dbjobs-template'] }}
[{{ section('dbjobs-cron-entry') }}]
recipe = slapos.cookbook:cron.d
cron-entries = ${cron:cron-entries}
name = dbjobs
frequency = * * * * *
command = ${dbjobs-executable:rendered}
[mysqld-restart-script]
< = jinja2-template-executable
rendered = ${directory:bin}/mysqld_restart
template = inline:#!/bin/sh
# This script is disabled on SlapOS
echo "RESTART FROM dbjbos WAS DISABLED"
[{{ section('mariadb-need-start') }}]
recipe = slapos.cookbook:cron.d
cron-entries = ${cron:cron-entries}
name = mariadb-need-start
frequency = * * * * *
command = ${template-mysqld-need-start:rendered}
[template-mysqld-need-start]
< = jinja2-template-executable
rendered = ${directory:bin}/mysqld_need_start
template = {{ parameter_dict['template-mysqld-need-start'] }}
context =
key mariadb_controller mariadb-ctl-bin:wrapper-path
key update_config mysql-get-config:rendered
raw username {{ slapparameter_dict['repman-user'] }}
raw repman_url {{ slapparameter_dict['repman-url'] }}
raw jq_bin {{ jq_bin }}
raw cluster {{ slapparameter_dict['cluster'] }}
raw db_host {{ host }}
raw db_port {{ port }}
raw bash_bin {{ bash_bin }}
raw curl_bin {{ curl_bin }}
# Donwnload mariadb configuration from repman
[mysql-get-config]
< = jinja2-template-executable
rendered = ${directory:bin}/mysqld-update-config
cluster = {{ slapparameter_dict['cluster'] }}
template = inline:#!{{ bash_bin }}
cd ${directory:config-tmp} &&
{{ curl_bin }} -o config.tar.gz {{ slapparameter_dict['repman-url'] }}/api/clusters/${:cluster}/servers/{{ host }}/{{ port }}/config
tar -xzf config.tar.gz
cp -r data/.system ${directory:mysql}
rm -rf ${directory:etc}/mysql
cp -r etc/mysql ${directory:etc}
ln -sf ${directory:mysql}/.system ${directory:var}/system
ln -sf ${my-cnf:rendered} ${directory:etc}/mysql/custom/01_mariadb.cnf
[{{ section('install-mysql-config') }}]
recipe = plone.recipe.command
stop-on-error = true
config = ${directory:etc}/mysql/my.cnf
command = ${mysql-get-config:rendered}
update-command = ${:command}
[dash]
dash = {{ dumps(dash) }}
[{{ section('promise-check-computer-memory') }}]
<= monitor-promise-base
module = check_command_execute
name = check-computer-memory.py
config-command = "{{ parameter_dict["check-computer-memory-binary"] }}" -db ${monitor-instance-parameter:collector-db} --threshold "{{ slapparameter_dict["computer-memory-percent-threshold"] }}" --unit percent
[{{ section('promise') }}]
<= monitor-promise-base
module = check_command_execute
name = mariadb.py
config-command = "{{ parameter_dict['bin-directory'] }}/is-local-tcp-port-opened" "{{ ip }}" "{{ port }}"
[monitor-instance-parameter]
monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }}
monitor-httpd-port = {{ port + 1 }}
monitor-title = {{ slapparameter_dict['name'] }}
password = {{ slapparameter_dict['monitor-passwd'] }}
[buildout]
extends =
{{ template_monitor }}
parts +=
{{ part_list | join('\n ') }}
software/repman/instance-repman-input-schema.json 0 → 100644
View file @ ce787d53
{
"type": "object",
"$schema": "http://json-schema.org/draft-04/schema",
"title": "Input Parameters",
"properties": {
"slave-frontend": {
"title": "Web frontend",
"description": "Front end used to provide web access for internal services at the kvm.",
"properties": {
"slave-domain": {
"title": "Slave frontend domain",
"description": "Unique domain name for this slave frontend.",
"type": "string",
"default": ""
},
"instance-guid": {
"title": "Main Frontend Instance ID",
"description": "Unique identifier of the frontend instance, like \"SOFTINST-11031\".",
"type": "string",
"default": ""
},
"frontend-software-type": {
"title": "Frontend Software Type",
"description": "Type of the frontend instance, like \"frontend\".",
"type": "string"
},
"frontend-software-url": {
"title": "Frontend Software URL",
"description": "Software Release URL of the frontend instance, like \"http://example.com/path/to/software.cfg\".",
"type": "string",
"format": "uri",
"default": "http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg"
}
},
"type": "object"
},
"computer-memory-percent-threshold": {
"title": "Computer memory percent threshold.",
"description": "Computer memory percent threshold.",
"type": "int",
"default": 80
},
"monitor-interface-url": {
"title": "Monitor Web Interface URL",
"description": "Give Url of HTML web interface that will be used to render this monitor instance.",
"type": "string",
"format": "uri",
"default": "https://monitor.app.officejs.com"
},
"monitor-cors-domains": {
"title": "Monitor CORS domains",
"description": "List of cors domains separated with space. Needed for ajax query on this monitor instance from a different domain.",
"type": "string",
"default": "monitor.app.officejs.com"
},
"mail-from": {
"title": "Mail From",
"description": "Mail From address",
"type": "string",
"default": ""
},
"mail-smtp-addr": {
"title": "Mail SMTP address",
"description": "Mail SMTP address. Default: localhost:25",
"type": "string",
"default": "localhost:25"
},
"mail-smtp-password": {
"title": "Mail SMTP password",
"description": "Mail SMTP password",
"type": "string",
"default": ""
},
"mail-smtp-user": {
"title": "Mail SMTP User",
"description": "Mail SMTP User",
"type": "string",
"default": ""
},
"mail-to": {
"title": "Mail To",
"description": "",
"type": "string",
"default": ""
},
"tags": {
"title": "Provisioning db tags",
"description": "Provisioning db tags",
"type": "string",
"default": "gtidstrict,bind,pkg,innodb,noquerycache,slow,pfs,linux,readonly,diskmonitor,sqlerror,compressbinlog,bm4ci,mroonga,utctime,readcommitted,nohandshake"
},
"http-session-lifetime": {
"title": "Web Session life time in s",
"description": "Web interface Session life time in seconds. Default 86400",
"type": "integer",
"default": 86400
},
"http-refresh-interval": {
"title": "Web refresh interval in s",
"description": "Web interface refresh interval in s. Default 4s.",
"type": "integer",
"default": 4
},
"autorejoin": {
"title": "Automatic rejoin a failed master",
"description": "Automatic rejoin a failed master (default true)",
"type": "boolean",
"default": true
},
"autoseed": {
"title": "Automatic join a standalone node",
"description": "Automatic join a standalone node",
"type": "boolean",
"default": true
},
"repman-cluster-dict": {
"title": "Replication Manager clusters definition",
"description": "Replication Manager clusters definition",
"patternProperties": {
".*": {
"properties": {
"name": {
"title": "Name of the cluster",
"description": "Name of the cluster: Should not contains spaces or any special characters.",
"type": "string",
"default": ""
},
"database-amount": {
"title": "Amount of databases for cluster",
"description": "Database amount to deploy with this cluster. Minimal amount is 2 required to enable replication.",
"type": "integer",
"default": 2,
"minimum": 2
},
"-sla-0-computer_guid": {
"title": "Prefered Master Computer Guid",
"description": "Computer Guid for prefered Master database.",
"type": "string",
"default": ""
},
"-sla-1-computer_guid": {
"title": "Slave 1 Computer Guid",
"description": "Computer Guid for the first slave database.",
"type": "string",
"default": ""
},
"-sla-2-computer_guid": {
"title": "Slave 2 Computer Guid",
"description": "Computer Guid for the second slave database.",
"type": "string",
"default": ""
},
"-sla-3-computer_guid": {
"title": "Slave 3 Computer Guid",
"description": "Computer Guid for the third slave database.",
"type": "string",
"default": ""
},
"-sla-4-computer_guid": {
"title": "Slave 4 Computer Guid",
"description": "Computer Guid for the fourth slave database.",
"type": "string",
"default": ""
},
"proxysql-user": {
"title": "Proxysql username",
"description": "Proxysql external user, default is 'external'.",
"type": "string",
"default": "external"
},
"proxy-tags": {
"title": "Proxy tags",
"description": "playbook configuration tags. Default: pkg,masterslave,linux,noreadwritesplit",
"type": "string",
"default": "pkg,masterslave,linux,noreadwritesplit"
},
"logical-backup-cron": {
"title": "Mysqldump backup cron definition.",
"description": "Logical backup cron expression represents a set of times, using cron format.",
"type": "string",
"default": "0 21 * * *"
},
"physical-backup-cron": {
"title": "Mariabackup cron definition",
"description": "Physical backup cron expression represents a set of times, using cron format.",
"type": "string",
"default": "0 1 * * *"
},
"proxy-cpu-cores": {
"title": "Proxy Cpu cores",
"description": "Proxy Cpu cores. Default: 2",
"type": "integer",
"default": 2,
"minimum": 1
},
"proxy-memory": {
"title": "Proxy Memory usage in giga bytes",
"description": "Proxy Memory usage in giga bytes. Default: 1G",
"type": "integer",
"default": 1,
"minimum": 1
},
"db-cpu-cores": {
"title": "Database Cpu cores",
"description": "Database Cpu cores. Default: 2",
"type": "integer",
"default": 2,
"minimum": 1
},
"db-disk-iops": {
"title": "Database Rnd IO/s",
"description": "Rnd IO/s in seconds for micro service VM (default 300).",
"type": "integer",
"default": 300
},
"db-memory": {
"title": "Database memory in M",
"description": "Memory in M for micro service VM (default 256)",
"type": "integer",
"default": 256,
"minimum": 256,
"multipleOf": 256
},
"db-memory-shared-pct": {
"title": "Percent memory shared per buffer",
"description": "Percent memory shared per buffer (default \"threads:16,innodb:60,myisam:10,aria:10,rocksdb:1,tokudb:1,s3:1,archive:1,querycache:0\")",
"type": "string",
"default": "threads:16,innodb:60,myisam:10,aria:10,rocksdb:1,tokudb:1,s3:1,archive:1,querycache:0"
},
"db-memory-threaded-pct": {
"title": "Percent memory allocted per threads",
"description": "Percent memory allocted per threads. (default \"tmp:70,join:20,sort:10\")",
"type": "string",
"default": "tmp:70,join:20,sort:10"
},
"innodb-file-per-table": {
"title": "enable Innodb file per table",
"description": "enable Innodb file per table. Possible value: 0=disabled, 1=enabled",
"type": "integer",
"default": 1,
"minimum": 0,
"maximum": 1
},
"use-ipv6": {
"title": "Mariadb server listen on IPv6",
"description": "Listen on IPv6 instead of IPv4.",
"type": "boolean",
"default": true
},
"failover-mode": {
"title": "Failover mode",
"description": "Failover is manual or automatic (default \"manual\").",
"type": "string",
"default": "manual",
"enum": [
"manual",
"automatic"
]
},
"failover-limit": {
"title": "Failover amount limit",
"description": "Failover is canceld if already failover this number of time (0: unlimited) (default 5).",
"type": "integer",
"default": 5
},
"failover-falsepositive-heartbeat": {
"title": "Failover check slaves do not receive heartbeat",
"description": "Failover checks that slaves do not receive heartbeat (default true).",
"type": "boolean",
"default": true
},
"failover-falsepositive-heartbeat-timeout": {
"title": "Failover check slaves do not receive heartbeat timeout",
"description": "Failover checks that slaves do not receive heartbeat detection timeout (default 3).",
"type": "integer",
"default": 3
},
"failover-falsepositive-ping-counter": {
"title": "Failover amount of ping failures",
"description": "Failover after this number of ping failures (interval 1s) (default 5).",
"type": "integer",
"default": 5
},
"failover-max-slave-delay": {
"title": "Failover election ignore slave with replication delay",
"description": "Election ignore slave with replication delay over this time in sec (default 30).",
"type": "integer",
"default": 30
},
"failover-readonly-state": {
"title": "Failover Switchover set slaves as read-only",
"description": "Failover Switchover set slaves as read-only (default true).",
"type": "boolean",
"default": true
},
"failover-restart-unsafe": {
"title": "Failover when cluster down if a slave is start first",
"description": "Failover when cluster down if a slave is start first.",
"type": "boolean",
"default": false
},
"failover-time-limit": {
"title": "Cancel failover if time in sec not passed after previous failover",
"description": "Failover is canceled if timer in sec is not passed with previous failover (0: do not wait).",
"type": "integer",
"default": 0
},
"switchover-at-equal-gtid": {
"title": "Switchover only when slaves are fully in sync",
"description": "Switchover only when slaves are fully in sync.",
"type": "boolean",
"default": false
},
"switchover-slave-wait-catch": {
"title": "Switchover wait for slave to catch with replication",
"description": "Switchover wait for slave to catch with replication, not needed in GTID mode but enable to detect possible issues like witing on old master (default true).",
"type": "boolean",
"default": true
},
"switchover-wait-kill": {
"title": "Switchover wait ms before killing threads on demoted master",
"description": "Switchover wait this many milliseconds before killing threads on demoted master (default 5000).",
"type": "integer",
"default": 5000
},
"switchover-wait-trx": {
"title": "Cancel switchover after timeout seconds if can't aquire FTWRL",
"description": "Switchover is cancel after this timeout in second if can't aquire FTWRL (default 10).",
"type": "integer",
"default": 10
},
"switchover-wait-write-query": {
"title": "Cancel switchover after timeout if a write query is running",
"description": "Switchover is canceled if a write query is running for this time (default 10).",
"type": "integer",
"default": 10
}
},
"type": "object"
}
},
"type": "object"
}
}
}
\ No newline at end of file
software/repman/instance-repman-output-schema.json 0 → 100644
View file @ ce787d53
{
"name": "Output Parameters",
"properties": {
"backend-url": {
"title": "Backend URL",
"description": "URL used to connect directly to backend without frontend. Requires IPv6.",
"type": "string",
"format": "uri"
},
"url": {
"title": "URL",
"description": "URL used to connect to the service.",
"type": "string",
"format": "uri"
},
"repman-password": {
"title": "Repman password",
"description": "Password for Replication Manager service.",
"type": "string"
}
}
}
software/repman/instance-repman.cfg.jinja2.in 0 → 100644
View file @ ce787d53
{% set publish_dict = {} -%}
{% set part_list = [] -%}
{% set monitor_base_url_dict = {} -%}
{% set mariadb_dict = {} -%}
{% set mariadb_server_list = [] -%}
{% set receiver_port_list = [] -%}
{% set mariadb_path_list = [] -%}
{% set ip = (ipv6_set | list)[0] -%}
{% set ipv4 = (ipv4_set | list)[0] -%}
{% set cluster_list = [] -%}
{% set tags = "gtidstrict,bind,pkg,innodb,noquerycache,slow,pfs,linux,readonly,diskmonitor,sqlerror,compressbinlog,bm4ci,mroonga,utctime,readcommitted,nohandshake" -%}
{% set frontend_parameter_dict = slapparameter_dict.get('slave-frontend', {}) -%}
[directory]
recipe = slapos.cookbook:mkdirectory
home = ${buildout:directory}
etc = ${:home}/etc
var = ${:home}/var
run = ${:var}/run
scripts = ${:etc}/run
service = ${:etc}/service
controller = ${:etc}/controller
promise = ${:etc}/promise
log = ${:var}/log
data = ${:var}/lib
nginx-prefix = ${:var}/nginx
tmp = ${:home}/tmp
{% import "supervisord_lib" as supervisord_lib with context %}
{% set proxysql_controller = "proxysql-ctl" -%}
{{ supervisord_lib.supervisord(proxysql_controller, buildout_bin_directory, supervisord_conf, use_service_hash=False) }}
{% do part_list.append("supervisord-proxysql-ctl") -%}
[request-common]
recipe = slapos.cookbook:request.serialised
software-url = ${slap-connection:software-release-url}
server-url = ${slap-connection:server-url}
key-file = ${slap-connection:key-file}
cert-file = ${slap-connection:cert-file}
computer-id = ${slap-connection:computer-id}
partition-id = ${slap-connection:partition-id}
[download-proxy-config]
recipe = slapos.recipe.template:jinja2
template = inline:#!{{ bash_bin }}
NAME=$1
HOST=$2
PORT=$3
CONFIG=$4
if [ -z "$CONFIG" ]; then
CONFIG="${repman:proxies}/proxysql-$NAME.cnf"
fi
mkdir -p ${repman:config-tmp}/proxies
cd ${repman:config-tmp}/proxies
{{ curl_bin }} -o proxies-$NAME.tar.gz ${nginx-parameter:repman-url}/api/clusters/$NAME/servers/$HOST/$PORT/config
tar -xzf proxies-$NAME.tar.gz
cp conf/proxysql.cnf $CONFIG
rendered = ${directory:bin}/update-proxysql-config
mode = 755
{% do mariadb_dict.__setitem__('computer-memory-percent-threshold', 80) -%}
{% set default_parameter_dict = {"cluster1": {"name": "cluster1", "db-prefered-master": "",
"database-amount": 2, "proxysql-user": "external", "proxy-tags": "pkg,masterslave,linux,noreadwritesplit",
"logical-backup-cron": "0 21 * * *", "physical-backup-cron": "0 1 * * *"}} -%}
{% for name, parameter_dict in slapparameter_dict.get('repman-cluster-dict', default_parameter_dict).items() -%}
{% do mariadb_dict.__setitem__('innodb-file-per-table', parameter_dict.get('innodb-file-per-table', 1)) -%}
{% do mariadb_dict.__setitem__('use-ipv6', parameter_dict.get('use-ipv6', True)) -%}
{% set database_list = parameter_dict.get('database-list', [{'name': 'repdb', 'user': 'user', 'password': 'insecure'}]) -%}
# Request mariadb instances
{% set db_amount = parameter_dict.get('database-amount', 2) -%}
{% if db_amount < 2 -%}
{% set db_amount = 2 -%}
{% endif -%}
{% for i in range(0, db_amount) -%}
{% do mariadb_dict.__setitem__('tcp-port', 2099 + (i * 100)) -%}
{% set section = 'request-mariadb-' ~ i -%}
{% set dbname = 'Mariadb-' ~ i -%}
[{{ section }}]
<= request-common
software-type = mariadb
name = {{ dbname }}
sla-computer_guid = {{ dumps(parameter_dict.get('-sla-' ~ i ~'-computer_guid', '')) }}
{% for key, value in mariadb_dict.items() -%}
config-{{ key }} = {{ dumps(value) }}
{% endfor -%}
config-monitor-passwd = ${publish-early:monitor-password}
config-root-password = ${publish-early:db-root-password}
config-repman-user = ${repman-parameter:username}
config-heartbeat-user = ${repman-parameter:heartbeat-user}
#config-repman-passwd = ${repman-parameter:password}
config-repman-url = ${nginx-parameter:backend-url}
config-repman-secure-url = ${nginx-parameter:backend-ssl-url}
config-cluster = {{ name }}
config-name = {{ dbname }}
config-database-list = {{ dumps(database_list) }}
return =
database-host
receiver-port
monitor-base-url
partition-path
{% do part_list.append(section) -%}
{% do mariadb_server_list.append('${' ~ section ~ ':connection-database-host}') -%}
{% do receiver_port_list.append('${' ~ section ~ ':connection-receiver-port}') -%}
{% do mariadb_path_list.append('${' ~ section ~ ':connection-partition-path}') -%}
{% do monitor_base_url_dict.__setitem__('mariadb' ~ i, '${' ~ section ~ ':connection-monitor-base-url}') -%}
{% endfor -%}
# Manage Replication Manager clusters
[{{name}}-admin-port]
recipe = slapos.cookbook:free_port
ip = {{ ipv4 }}
minimum = 6032
maximum = 6132
[{{name}}-port]
recipe = slapos.cookbook:free_port
ip = {{ ipv4 }}
minimum = 7032
maximum = 7132
{% set prefered_master = parameter_dict.get("db-prefered-master") -%}
[{{ name ~ '-cluster-parameter' }}]
{% for key, value in parameter_dict.items() -%}
{{ key }} = {{ value }}
{% endfor -%}
proxysql-user = {{ parameter_dict.get("proxysql-user", "external") }}
proxy-port = {{ '${' ~ name ~ '-port:port}' }}
proxy-admin-port = {{ '${' ~ name ~ '-admin-port:port}' }}
db-user = repman
db-password = ${publish-early:db-root-password}
db-list = {{ mariadb_server_list | join(',') }}
heartbeat-user = ${repman-parameter:heartbeat-user}
heartbeat-password = ${publish-early:db-root-password}
partition-list = {{ mariadb_path_list | join(',') }}
{% if prefered_master -%}
db-prefered-master = {{ prefered_master }}
{% else -%}
# First database is the prefered master
db-prefered-master = {{ mariadb_server_list[0] }}
{% endif -%}
proxysql-servers = {{ ipv4 }}
proxysql-servers-ipv6 = [{{ ip }}]
password = ${repman-parameter:password}
proxysql-partition = ${buildout:directory}
receiver-port-list = {{ receiver_port_list | join(',') }}
proxy-tags = {{ parameter_dict.get("proxy-tags", "pkg,masterslave,linux,noreadwritesplit") }}
logical-backup-cron = {{ parameter_dict.get("logical-backup-cron", "0 22 * * *") }}
physical-backup-cron = {{ parameter_dict.get("physical-backup-cron", "0 0 * * *") }}
proxy-cpu-cores = {{ parameter_dict.get("proxy-cpu-cores", 2) }}
proxy-memory = {{ parameter_dict.get("proxy-memory", 1) }}
db-cpu-cores = {{ parameter_dict.get("db-cpu-cores", 2) }}
db-disk-iops = {{ parameter_dict.get("db-disk-iops", 300) }}
db-memory = {{ parameter_dict.get("db-memory", 256) }}
db-memory-shared-pct = {{ parameter_dict.get("db-memory-shared-pct", "threads:16,innodb:60,myisam:10,aria:10,rocksdb:1,tokudb:1,s3:1,archive:1,querycache:0") }}
db-memory-threaded-pct = {{ parameter_dict.get("db-memory-threaded-pct", "tmp:70,join:20,sort:10") }}
# failover
failover-mode = {{ parameter_dict.get('failover-mode', 'manual') }}
failover-limit = {{ parameter_dict.get('failover-limit', 5) }}
failover-falsepositive-heartbeat = {{ parameter_dict.get('failover-falsepositive-heartbeat', True) }}
failover-falsepositive-heartbeat-timeout = {{ parameter_dict.get('failover-falsepositive-heartbeat-timeout', 3) }}
failover-falsepositive-ping-counter = {{ parameter_dict.get('failover-falsepositive-ping-counter', 5) }}
failover-max-slave-delay = {{ parameter_dict.get('failover-max-slave-delay', 30) }}
failover-readonly-state = {{ parameter_dict.get('failover-readonly-state', True) }}
failover-restart-unsafe = {{ parameter_dict.get('failover-restart-unsafe', False) }}
failover-time-limit = {{ parameter_dict.get('failover-time-limit', 0) }}
#switchover
switchover-at-equal-gtid = {{ parameter_dict.get('switchover-at-equal-gtid', False) }}
switchover-slave-wait-catch = {{ parameter_dict.get('switchover-slave-wait-catch', True) }}
switchover-wait-kill = {{ parameter_dict.get('switchover-wait-kill', 5000) }}
switchover-wait-trx = {{ parameter_dict.get('switchover-wait-trx', 10) }}
switchover-wait-write-query = {{ parameter_dict.get('switchover-wait-write-query', 10) }}
[{{ 'config-' ~ name }}]
recipe = slapos.recipe.template:jinja2
template = {{ config_cluster_toml_in }}
rendered = ${repman:clusters}/config-{{ name }}.toml
extra-context =
context =
section parameter_dict {{ name ~ '-cluster-parameter' }}
# Donwnload mariadb configuration from repman
[config-proxysql-{{ name }}]
recipe = plone.recipe.command
# if Repman is not started, cannot download config from server
stop-on-error = false
config = ${repman:proxies}/proxysql-{{ name }}.cnf
data = ${repman:proxy-data}/{{ name }}
command =
mkdir -p ${:data} &&
${download-proxy-config:rendered} {{ name }} {{ ipv4 }} {{ '${' ~ name ~ '-cluster-parameter:proxy-admin-port}' }} ${:config}
update-command = ${:command}
[proxysql-{{ name }}-wrapper]
recipe = slapos.cookbook:wrapper
command-line =
{{ proxysql_location }}/bin/proxysql -f
-c ${config-proxysql-{{ name }}:config}
-D ${config-proxysql-{{ name }}:data}
--reload
# -S /tmp/proxysql_admin.sock
wrapper-path = ${directory:controller}/proxysql-{{ name }}
wait-for-files =
${repman:bootstrap}/{{ name }}_bootstrapped
depends =
{{ '${proxysql-' ~ name ~ '-admin-promise:recipe}' }}
{{ '${proxysql-' ~ name ~ '-promise:recipe}' }}
{{ '${proxysql-' ~ name ~ '-ipv6-promise:recipe}' }}
[proxysql-{{ name }}-admin-promise]
<= monitor-promise-base
module = check_port_listening
name = proxysql-{{ name }}-admin-port-listening.py
config-hostname= {{ ipv4 }}
config-port = {{ '${' ~ name ~ '-cluster-parameter:proxy-admin-port}' }}
[proxysql-{{ name }}-promise]
<= monitor-promise-base
module = check_port_listening
name = proxysql-{{ name }}-port-listening.py
config-hostname= {{ ipv4 }}
config-port = {{ '${' ~ name ~ '-cluster-parameter:proxy-port}' }}
[proxysql-{{ name }}-ipv6-promise]
<= monitor-promise-base
module = check_port_listening
name = proxysql-{{ name }}-ipv6-port-listening.py
config-hostname= {{ ip }}
config-port = {{ '${' ~ name ~ '-cluster-parameter:proxy-port}' }}
{% set service_name = "proxysql-" ~ name -%}
{% set proxysql_dict = {"name": service_name, "command": "${" ~ service_name ~ "-wrapper:wrapper-path}",
"stopwaitsecs": 60, "environment": [],
"stdout_logfile": "${repman:proxies-log}/" ~ service_name ~ ".log",
"stderr_logfile": "${repman:proxies-log}/" ~ service_name ~ ".log" } %}
{{ supervisord_lib.supervisord_program(service_name, proxysql_dict) }}
{% do part_list.append("supervisord-" ~ service_name) %}
{% do part_list.append('config-' ~ name) -%}
{% do cluster_list.append("{'name': '" ~ name ~ "', 'host': '" ~ ipv4 ~ "', 'port': '${" ~ name ~ "-cluster-parameter:proxy-admin-port}'}") -%}
{% set publish_database_list = [] -%}
{% set publish_database_v6_list = [] -%}
{% for database in database_list -%}
{% if database.get('user') -%}
{% do publish_database_list.append("mysql://" ~ database['user'] ~ ":" ~ database['password'] ~ "@" ~ ipv4 ~ ":${" ~ name ~ "-cluster-parameter:proxy-port}/" ~ database['name']) -%}
{% do publish_database_v6_list.append("mysql://" ~ database['user'] ~ ":" ~ database['password'] ~ "@[" ~ ip ~ "]:${" ~ name ~ "-cluster-parameter:proxy-port}/" ~ database['name']) -%}
{% else -%}
{% do publish_database_list.append("mysql://" ~ ipv4 ~ ":${" ~ name ~ "-cluster-parameter:proxy-port}/" ~ database['name']) -%}
{% do publish_database_v6_list.append("mysql://[" ~ ip ~ "]:${" ~ name ~ "-cluster-parameter:proxy-port}/" ~ database['name']) -%}
{% endif -%}
{% endfor -%}
{% do publish_dict.__setitem__(name ~ '-database-list', "!py!['" ~ publish_database_list | join("', '") ~ "']") -%}
{% do publish_dict.__setitem__(name ~ '-database-list-v6', "!py!['" ~ publish_database_v6_list | join("', '") ~ "']") -%}
{% endfor -%}
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration
computer = ${slap-connection:computer-id}
partition = ${slap-connection:partition-id}
url = ${slap-connection:server-url}
key = ${slap-connection:key-file}
cert = ${slap-connection:cert-file}
[instance-parameter]
<= slap-configuration
# repman monitor seems to use a fixed port
repman-secure-port = 10005
repman-port = 10001
nginx-ssl-port = 10006
nginx-port = 10007
[repman]
recipe = slapos.cookbook:mkdirectory
etc = ${directory:etc}/repman
data-dir = ${directory:var}/lib
root-dir = ${directory:srv}/repman
clusters = ${:etc}/cluster.d
proxies = ${:etc}/proxy
proxy-data = ${:data-dir}/proxy
config-tmp = ${directory:tmp}/config
bootstrap = ${:etc}/bootstrap
proxies-log = ${directory:log}/proxy
[nginx-parameter]
ipv6 = ${instance-parameter:ipv6-random}
port = ${instance-parameter:nginx-port}
ssl-port = ${instance-parameter:nginx-ssl-port}
ssl-certificate = ${ca-nginx:cert-file}
ssl-key = ${ca-nginx:key-file}
pid-file = ${directory:run}/nginx.pid
access-log = ${directory:log}/nginx_access.log
error-log = ${directory:log}/nginx_error.log
repman-secure-url = https://${repman-parameter:ipv4}:${repman-parameter:secure-port}
repman-url = http://${repman-parameter:ipv4}:${repman-parameter:port}
config-file = ${directory:etc}/nginx.conf
backend-ssl-url = https://[${:ipv6}]:${:ssl-port}
backend-url = http://[${:ipv6}]:${:port}
[repman-password]
recipe = slapos.cookbook:generate.password
bytes = 12
[gen-root-password]
recipe = slapos.cookbook:generate.password
bytes = 12
[repman-parameter]
log = ${directory:log}/repman.log
http-root = ${repman:root-dir}/dashboard
share-dir = ${repman:root-dir}/share
secure-port = ${instance-parameter:repman-secure-port}
port = ${instance-parameter:repman-port}
ipv4 = ${instance-parameter:ipv4-random}
username = admin
heartbeat-user = heartbeat
password = ${publish-early:repman-password}
cluster-d = ${repman:clusters}
autorejoin = {{ slapparameter_dict.get("autorejoin", True) }}
autoseed = {{ slapparameter_dict.get("autoseed", True) }}
mysql-bin-dir = {{ mariadb_location }}/bin
mysqlbinlog-path = {{ mariadb_location }}/bin/mysqlbinlog
mysqlclient-path = {{ mariadb_location }}/bin/mysql
mysqldump-path = {{ mariadb_location }}/bin/mysqldump
haproxy-bin = {{ haproxy_location }}/sbin/haproxy
sysbench-bin = {{ sysbench_location }}/bin/sysbench
restic-bin = {{ restic_bin_location }}
mail-from = {{ slapparameter_dict.get("mail-from", "mrm@localhost") }}
mail-smtp-addr = {{ slapparameter_dict.get("mail-smtp-addr", "localhost:25") }}
mail-smtp-password = {{ slapparameter_dict.get("mail-smtp-password", "") }}
mail-smtp-user = {{ slapparameter_dict.get("mail-smtp-user", "") }}
mail-to = {{ slapparameter_dict.get("mail-to", "") }}
enabled-tags = {{ slapparameter_dict.get("tags", tags) }}
http-session-lifetime = {{ slapparameter_dict.get("http-session-lifetime", 86400) }}
http-refresh-interval = {{ slapparameter_dict.get("http-refresh-interval", 4) }}
[repman-config-folder]
recipe = plone.recipe.command
repman-location = {{ repman_src_location }}
command =
cd ${:repman-location}
{{ rsync_location }}/bin/rsync -av share ${repman:root-dir}/
{{ rsync_location }}/bin/rsync -av dashboard ${repman:root-dir}/
update-command = ${:command}
[replication-manager-reload]
recipe = slapos.recipe.template:jinja2
template = {{ template_repman_manager_sh }}
cluster-list = {{ dumps(slapparameter_dict.get('repman-cluster-dict', default_parameter_dict).keys() ) }}
context =
section parameter_dict repman
key username repman-parameter:username
key password repman-parameter:password
key secure_url nginx-parameter:backend-ssl-url
key cluster_name_list :cluster-list
raw jq_bin {{ jq_bin }}
raw curl_bin {{ curl_bin }}
raw bash_bin {{ bash_bin }}
rendered = ${directory:scripts}/repman-reload
mode = 755
[replication-manager]
recipe = slapos.cookbook:wrapper
command-line =
{{ gowork_bin }}/replication-manager
--monitoring-basedir=${repman:root-dir}
--monitoring-sharedir=${repman-parameter:share-dir}
--http-root=${repman-parameter:http-root}
--monitoring-datadir=${repman:data-dir}
--config=${repman-config.toml:rendered}
--log-file=${repman-parameter:log}
--memprofile=${directory:tmp}/repmgr.mprof
monitor
wrapper-path = ${directory:service}/replication-manager
# setup repman instance folder
depends =
${repman-config-folder:recipe}
${replication-manager-reload:recipe}
${repman-listen-promise:recipe}
${repman-listen-ssl-promise:recipe}
[repman-config.toml]
recipe = slapos.recipe.template:jinja2
template = {{ config_toml_in }}
rendered = ${repman:etc}/config.toml
extra-context =
context =
section parameter_dict repman-parameter
[repman-listen-promise]
<= monitor-promise-base
module = check_port_listening
name = repman_service_listen.py
config-hostname = ${repman-parameter:ipv4}
config-port = ${repman-parameter:port}
[repman-listen-ssl-promise]
<= monitor-promise-base
module = check_port_listening
name = repman_service_ssl_listen.py
config-hostname = ${repman-parameter:ipv4}
config-port = ${repman-parameter:secure-port}
[nginx-conf]
recipe = slapos.recipe.template:jinja2
template = {{ nginx_conf_in }}
rendered = ${nginx-parameter:config-file}
context =
section parameter_dict nginx-parameter
[nginx-launcher]
recipe = slapos.cookbook:wrapper
command-line =
{{ nginx_bin }}
-p ${directory:nginx-prefix}
-c ${nginx-conf:rendered}
wrapper-path = ${directory:bin}/nginx-start
wait-for-files =
${ca-directory:certs}/nginx.key
${ca-directory:certs}/nginx.crt
${nginx-graceful-wrapper:rendered}
[nginx-graceful-wrapper]
recipe = slapos.recipe.template:jinja2
template = inline:#!{{ bash_bin }}
kill -USR1 "$(cat ${nginx-parameter:pid-file})"
rendered = ${directory:scripts}/nginx-graceful
context =
mode = 755
[ca-nginx]
<= certificate-authority
recipe = slapos.cookbook:certificate_authority.request
cert-file = ${ca-directory:certs}/nginx.crt
key-file = ${ca-directory:certs}/nginx.key
executable = ${nginx-launcher:wrapper-path}
wrapper = ${directory:bin}/ca-nginx
[ca-nginx-service]
recipe = slapos.cookbook:wrapper
command-line = ${ca-nginx:wrapper}
wrapper-path = ${directory:services}/nginx
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
[logrotate-entry-nginx]
<= logrotate-entry-base
name = nginx
log = ${nginx-parameter:access-log} ${nginx-parameter:error-log}
post = kill -USR1 $(cat ${nginx-parameter:pid-file})
[publish-early]
recipe = slapos.cookbook:publish-early
-init =
monitor-password monitor-htpasswd:passwd
db-root-password gen-root-password:passwd
repman-password repman-password:passwd
[publish-connection-parameter]
<= monitor-publish
-extends = publish-early
recipe = slapos.cookbook:publish
backend-url = ${nginx-parameter:backend-ssl-url}
url = ${repman-frontend:connection-secure_access}
username = ${repman-parameter:username}
{% for name, value in publish_dict.items() -%}
{{ name }} = {{ value }}
{% endfor %}
[monitor-instance-parameter]
monitor-httpd-port = 8060
cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.app.officejs.com') }}
username = admin
password = ${publish-early:monitor-password}
[monitor-base-url-dict]
{% for key, value in monitor_base_url_dict.items() -%}
{{ key }} = {{ value }}
{% endfor %}
[repman-frontend]
<= slap-connection
recipe = slapos.cookbook:requestoptional
name = Replication Manager Frontend
# XXX We have hardcoded SR URL here.
software-url = {{ frontend_parameter_dict.get('frontend-software-url', 'http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg') }}
{% if frontend_parameter_dict.get('frontend-software-type', '') -%}
software-type ={{ frontend_parameter_dict['frontend-software-type'] }}
{% endif -%}
slave = true
config-url = ${nginx-parameter:backend-ssl-url}
config-domain = {{ frontend_parameter_dict.get('slave-domain', '') }}
return = domain secure_access
[repman-frontend-promise]
<= monitor-promise-base
module = check_url_available
name = check_repman_frontend.py
config-url = https://${repman-frontend:connection-domain}
config-check-secure = 1
[repman-backend-promise]
<= monitor-promise-base
module = check_url_available
name = check_repman_backend.py
config-url = ${nginx-parameter:backend-ssl-url}
config-check-secure = 1
[template-proxysql-need-stop-start]
recipe = slapos.recipe.template:jinja2
rendered = ${directory:bin}/proxysql_check_stop_start
template = {{ template_proxy_need_stopstart }}
mode = 755
cluster-list = !py![{{ cluster_list | join(', ') }}]
context =
key proxysql_controller {{proxysql_controller}}-bin:wrapper-path
key repman_url nginx-parameter:backend-url
key get_proxy_config download-proxy-config:rendered
key cluster_list :cluster-list
raw jq_bin {{ jq_bin }}
raw bash_bin {{ bash_bin }}
raw curl_bin {{ curl_bin }}
[proxy-need-stop-start]
recipe = slapos.cookbook:cron.d
cron-entries = ${cron:cron-entries}
name = proxysql-need-stop-start
frequency = * * * * *
command = ${template-proxysql-need-stop-start:rendered}
#############################
#
# Deploy replication-manager instance
#
#############################
[buildout]
extends =
{{ template_monitor }}
parts =
replication-manager
monitor-base
logrotate-entry-nginx
ca-nginx-service
publish-connection-parameter
repman-frontend-promise
repman-backend-promise
proxy-need-stop-start
# Complete parts with sections
{{ part_list | join('\n ') }}
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
software/repman/instance.cfg.in 0 → 100644
View file @ ce787d53
[buildout]
parts = switch-softwaretype
eggs-directory = {{ buildout_egg_directory }}
develop-eggs-directory = {{ buildout_develop_directory }}
offline = true
[switch-softwaretype]
recipe = slapos.cookbook:switch-softwaretype
default = template-instance-repman.cfg:rendered
RootSoftwareInstance = ${:default}
mariadb = template-instance-mariadb.cfg:rendered
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
computer = ${slap-connection:computer-id}
partition = ${slap-connection:partition-id}
url = ${slap-connection:server-url}
key = ${slap-connection:key-file}
cert = ${slap-connection:cert-file}
[jinja2-template-base]
recipe = slapos.recipe.template:jinja2
mode = 0644
extensions = jinja2.ext.do
rendered= ${buildout:directory}/${:_buildout_section_name_}
supervisord-lib = {{ supervisord_lib }}
import-list =
file supervisord_lib :supervisord-lib
context =
key slapparameter_dict slap-configuration:configuration
key computer_id slap-configuration:computer
key ipv4_set slap-configuration:ipv4
key ipv6_set slap-configuration:ipv6
raw buildout_directory {{ buildout_directory }}
raw buildout_bin_directory {{ buildout_bin_directory }}
raw eggs_directory {{ buildout_egg_directory }}
raw develop_eggs_directory {{ buildout_develop_directory }}
raw mariadb_location {{ mariadb_location }}
raw supervisord_lib {{ supervisord_lib }}
raw supervisord_conf {{ supervisord_conf }}
raw template_monitor {{ template_monitor_cfg }}
# program binaries
raw bash_bin {{ bash_location }}/bin/bash
raw jq_bin {{ jq_location }}/jq
raw curl_bin {{ curl_location }}/bin/curl
${:extra-context}
extra-context =
[template-instance-repman.cfg]
<= jinja2-template-base
template= {{ template_repman_cfg }}
extra-context =
raw gowork_bin {{ gowork_bin }}
raw haproxy_location {{ haproxy_location }}
raw nginx_bin {{ nginx_location }}/sbin/nginx
raw repman_src_location {{ repman_src_location }}
# config files
raw config_toml_in {{ config_toml_in }}
raw config_cluster_toml_in {{ config_cluster_toml_in }}
raw nginx_conf_in {{ nginx_conf_in }}
raw rsync_location {{ rsync_location }}
raw restic_bin_location {{ restic_bin_location }}
raw sysbench_location {{ sysbench_location }}
raw proxysql_location {{ proxysql_location }}
raw template_repman_manager_sh {{ template_repman_manager_sh }}
raw template_proxy_need_stopstart {{ proxy_need_stop_start_template }}
[template-mariadb-parameters]
bash = {{ bash_location }}
dash-location = {{ dash_location }}
gzip-location = {{ gzip_location }}
mariadb-location = {{ mariadb_location }}
template-my-cnf = {{ template_my_cnf }}
template-mariadb-initial-setup = {{ template_mariadb_initial_setup }}
template-mariadb-init-root = {{ template_init_root_sql }}
template-init-root-wrapper = {{ template_init_root_wrapper }}
template-mysqld-wrapper = {{ template_mysqld_wrapper }}
template-mysqld-need-start = {{ mysqld_start_template }}
link-binary = {{ dumps(mariadb_link_binary) }}
check-computer-memory-binary = {{ bin_directory }}/check-computer-memory
bin-directory = {{ bin_directory }}
percona-tools-location = {{ percona_toolkit_location }}
unixodbc-location = {{ unixodbc_location }}
curl-location = {{ curl_location }}
dbjobs-template = {{ dbjobs_in }}
socat-location = {{ socat_location }}
mroonga-mariadb-install-sql = {{ mroonga_mariadb_install_sql }}
mroonga-mariadb-plugin-dir = {{ mroonga_mariadb_plugin_dir }}
groonga-plugins-path = {{ groonga_plugin_dir }}:{{ groonga_mysql_normalizer_plugin_dir }}
[template-instance-mariadb.cfg]
<= jinja2-template-base
template = {{ template_mariadb }}
filename = instance-mariadb.cfg
extra-context =
section parameter_dict template-mariadb-parameters
software/repman/software.cfg 0 → 100644
View file @ ce787d53
[buildout]
extends =
buildout.hash.cfg
../../component/restic/buildout.cfg
../../component/replication-manager/buildout.cfg
../../component/mariadb/buildout.cfg
../../component/nginx/buildout.cfg
../../component/haproxy/buildout.cfg
../../component/logrotate/buildout.cfg
../../component/percona-toolkit/buildout.cfg
../../component/gzip/buildout.cfg
../../component/sed/buildout.cfg
../../component/coreutils/buildout.cfg
../../component/grep/buildout.cfg
../../component/sysbench/buildout.cfg
../../component/proxysql/buildout.cfg
../../component/socat/buildout.cfg
../../component/rsync/buildout.cfg
../../stack/supervisord/buildout.cfg
../../stack/monitor/buildout.cfg
../neoppod/software-common.cfg
parts =
slapos-cookbook
mroonga-mariadb
instance.cfg
template-mariadb.cfg
template-mysqld-wrapper
gowork
[instance.cfg]
recipe = slapos.recipe.template:jinja2
rendered = ${buildout:directory}/instance.cfg
template = ${:_profile_base_location_}/${:filename}
mode = 0644
context =
key bash_location bash:location
key bin_directory buildout:bin-directory
key config_toml_in config-toml.in:target
key config_cluster_toml_in config-cluster-toml.in:target
key coreutils_location coreutils:location
key curl_location curl:location
key buildout_egg_directory buildout:eggs-directory
key buildout_develop_directory buildout:develop-eggs-directory
key buildout_directory buildout:directory
key buildout_bin_directory buildout:bin-directory
key dbjobs_in dbjobs-in:target
key dash_location dash:location
key jq_location jq-binary:location
key logrotate_cfg template-logrotate-base:rendered
key gowork_bin gowork:bin
key gzip_location gzip:location
key haproxy_location haproxy:location
key template_monitor monitor2-template:rendered
key mariadb_link_binary template-mariadb.cfg:link-binary
key mariadb_location mariadb:location
key mysqld_start_template mysqld-need-start.sh.in:target
key mroonga_mariadb_install_sql mroonga-mariadb:install-sql
key mroonga_mariadb_plugin_dir mroonga-mariadb:plugin-dir
key groonga_plugin_dir groonga:groonga-plugin-dir
key groonga_mysql_normalizer_plugin_dir groonga-normalizer-mysql:groonga-plugin-dir
key nginx_conf_in nginx.conf.in:target
key nginx_location nginx:location
key percona_toolkit_location percona-toolkit:location
key proxy_need_stop_start_template proxy-need-start-stop.sh.in:target
key repman_src_location git.signal18.io_signal18_repman:location
key rsync_location rsync:location
key restic_bin_location restic:location
key socat_location socat:location
key supervisord_lib supervisord-library:target
key supervisord_conf supervisord-conf:target
key template_repman_manager_sh repman-manager-sh.in:target
key template_mariadb template-mariadb.cfg:target
key template_mariadb_initial_setup template-mariadb-initial-setup:target
key template_monitor_cfg monitor2-template:rendered
key template_my_cnf template-my-cnf:target
key template_mysqld_wrapper template-mysqld-wrapper:rendered
key template_init_root_sql mariadb-init-root-sql:target
key template_init_root_wrapper init-root-wrapper-in:target
key template_repman_cfg instance-repman.cfg:target
key unixodbc_location unixodbc:location
key sysbench_location sysbench:location
key proxysql_location proxysql:location
[jq-binary]
recipe = hexagonit.recipe.download
url = https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64
md5sum = 1fffde9f3c7944f063265e9a5e67ae4f
filename = jq
mode = 0755
download-only = true
[download-file]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_update_hash_filename_}
destination = ${buildout:directory}/${:_buildout_section_name_}
[config-toml.in]
<= download-file
[config-cluster-toml.in]
<= download-file
[instance-repman.cfg]
<= download-file
[repman-manager-sh.in]
<= download-file
[template-mariadb.cfg]
<= download-file
link-binary =
${coreutils:location}/bin/basename
${coreutils:location}/bin/cat
${coreutils:location}/bin/cp
${coreutils:location}/bin/ls
${coreutils:location}/bin/tr
${coreutils:location}/bin/uname
${gettext:location}/lib/gettext/hostname
${grep:location}/bin/grep
${sed:location}/bin/sed
${mariadb:location}/bin/mysqlbinlog
[template-mariadb-initial-setup]
<= download-file
[template-my-cnf]
<= download-file
[mariadb-init-root-sql]
<= download-file
[init-root-wrapper-in]
<= download-file
[nginx.conf.in]
<= download-file
[dbjobs-in]
<= download-file
[mysqld-need-start.sh.in]
<= download-file
[proxy-need-start-stop.sh.in]
<= download-file
# Pin versions of eggs used that are not already pinned by stack/slapos.cfg
[versions]
slapos.recipe.template = 4.4
rubygemsrecipe = 0.2.2+slapos001
software/repman/software.cfg.json 0 → 100644
View file @ ce787d53
{
"name": "Replication Manager",
"description": "Replication Manager",
"serialisation": "xml",
"software-type": {
"default": {
"title": "Default",
"serialisation": "json-in-xml",
"description": "Replication Manager",
"request": "instance-repman-input-schema.json",
"response": "instance-repman-output-schema.json",
"index": 0
}
}
}
software/repman/templates/cluster-config.toml.in 0 → 100644
View file @ ce787d53
{% macro setbool(value) -%}
{% if value in ['true', 'True'] %} true {% else %} false {% endif -%}
{% endmacro -%}
[{{ parameter_dict['name'] }}]
title = "{{ parameter_dict['name'] }}"
monitoring-save-config = true
db-servers-hosts = "{{ parameter_dict['db-list'] }}"
db-servers-prefered-master = "{{ parameter_dict['db-prefered-master'] }}"
db-servers-credential = "{{ parameter_dict['db-user'] }}:{{ parameter_dict['db-password'] }}"
replication-credential = "{{ parameter_dict['db-user'] }}:{{ parameter_dict['db-password'] }}"
monitoring-write-heartbeat-credential="{{ parameter_dict['heartbeat-user'] }}:{{ parameter_dict['heartbeat-password'] }}"
db-servers-connect-timeout = 1
slapos-db-partitions = "{{ parameter_dict['partition-list'] }}"
slapos-proxysql-partitions = "{{ parameter_dict['proxysql-partition'] }}"
proxysql = true
proxysql-port = {{ parameter_dict['proxy-port'] }}
proxysql-servers = "{{ parameter_dict['proxysql-servers'] }}"
proxysql-servers-ipv6 = "{{ parameter_dict['proxysql-servers-ipv6'] }}"
proxysql-user = "{{ parameter_dict['proxysql-user'] }}"
proxysql-bootstrap = true
proxysql-admin-port = {{ parameter_dict['proxy-admin-port'] }}
proxysql-password = "{{ parameter_dict['password'] }}"
prov-proxy-tags = "{{ parameter_dict['proxy-tags'] }}"
monitoring-scheduler = true
scheduler-db-servers-logical-backup = true
scheduler-db-servers-logical-backup-cron = "0 {{ parameter_dict['logical-backup-cron'] }}"
scheduler-db-servers-logs = true
scheduler-db-servers-logs-cron = "0 0 23 * * *"
scheduler-db-servers-logs-table-keep = 4
scheduler-db-servers-logs-table-rotate = true
scheduler-db-servers-logs-table-rotate-cron = "0 0 23 * * *"
scheduler-db-servers-optimize = true
scheduler-db-servers-optimize-cron = "0 0 3 1 * 5"
scheduler-db-servers-physical-backup = true
scheduler-db-servers-physical-backup-cron = "0 {{ parameter_dict['physical-backup-cron'] }}"
backup-physical-type = "mariabackup"
backup-logical-type = "mysqldump"
scheduler-db-servers-receiver-ports= "{{ parameter_dict['receiver-port-list'] }}"
prov-proxy-cpu-cores = {{ parameter_dict['proxy-cpu-cores'] }}
prov-proxy-memory = {{ parameter_dict['proxy-memory'] }}
prov-db-cpu-cores = {{ parameter_dict['db-cpu-cores'] }}
prov-db-disk-iops = {{ parameter_dict['db-disk-iops'] }}
prov-db-memory = {{ parameter_dict['db-memory'] }}
prov-db-memory-shared-pct = "{{ parameter_dict['db-memory-shared-pct'] }}"
prov-db-memory-threaded-pct = "{{ parameter_dict['db-memory-threaded-pct'] }}"
test-inject-traffic = true
# failover
failover-mode = "{{ parameter_dict['failover-mode'] }}"
failover-limit = {{ parameter_dict['failover-limit'] }}
failover-falsepositive-heartbeat = {{ setbool(parameter_dict['failover-falsepositive-heartbeat']) }}
failover-falsepositive-heartbeat-timeout = {{ parameter_dict['failover-falsepositive-heartbeat-timeout'] }}
failover-falsepositive-ping-counter = {{ parameter_dict['failover-falsepositive-ping-counter'] }}
failover-max-slave-delay = {{ parameter_dict['failover-max-slave-delay'] }}
failover-readonly-state = {{ setbool(parameter_dict['failover-readonly-state']) }}
failover-restart-unsafe = {{ setbool(parameter_dict['failover-restart-unsafe']) }}
failover-time-limit = {{ parameter_dict['failover-time-limit'] }}
#switchover
switchover-at-equal-gtid = {{ setbool(parameter_dict['switchover-at-equal-gtid']) }}
switchover-slave-wait-catch = {{ setbool(parameter_dict['switchover-slave-wait-catch']) }}
switchover-wait-kill = {{ parameter_dict['switchover-wait-kill'] }}
switchover-wait-trx = {{ parameter_dict['switchover-wait-trx'] }}
switchover-wait-write-query = {{ parameter_dict['switchover-wait-write-query'] }}
software/repman/templates/config.toml.in 0 → 100644
View file @ ce787d53
{% macro setbool(value) -%}
{% if value in ['true', 'True'] %} true {% else %} false {% endif -%}
{% endmacro -%}
[Default]
api-bind = "{{ parameter_dict['ipv4'] }}"
http-bind-address = "{{ parameter_dict['ipv4'] }}"
http-server = true
http-session-lifetime = {{ parameter_dict['http-session-lifetime'] }}
http-refresh-interval = {{ int(parameter_dict['http-refresh-interval'])*1000 }}
monitoring-save-config = false
api-https-bind = true
api-credentials = "{{ parameter_dict['username'] }}:{{ parameter_dict['password'] }}"
include = "{{ parameter_dict['cluster-d'] }}"
autorejoin = {{ setbool(parameter_dict['autorejoin']) }}
autoseed = {{ setbool(parameter_dict['autoseed']) }}
{% if parameter_dict['autoseed'] in ['true', 'True'] -%}
autorejoin-logical-backup = true
{% endif -%}
db-servers-binary-path = "{{ parameter_dict['mysql-bin-dir'] }}"
# Database list of hosts to ignore in election
#db-servers-ignored-hosts =
# Database hosts list to monitor, IP and port (optional), specified in the host:[port] format and separated by commas
monitoring-address = "{{ parameter_dict['ipv4'] }}"
monitoring-wait-retry = 40
#haproxy = true
#haproxy-binary-path = "{{ parameter_dict['haproxy-bin'] }}"
# HaProxy input bind address for read (default "0.0.0.0")
#haproxy-ip-read-bind =
# HaProxy input bind address for write (default "0.0.0.0")
#haproxy-ip-write-bind =
# HaProxy load balance read port to all nodes (default 3307)
#haproxy-read-port =
# HaProxy hosts (default "127.0.0.1")
#haproxy-servers =
# HaProxy statistics port (default 1988)
#haproxy-stat-port =
#HaProxy read-write port to leader (default 3306)
#haproxy-write-port =
# Use restic to archive and restore backups
backup = true
backup-restic = true
backup-restic-binary-path = "{{ parameter_dict['restic-bin'] }}"
backup-restic-aws = false
backup-restic-password = "{{ parameter_dict['password'] }}"
backup-mysqlclient-path = "{{ parameter_dict['mysqlclient-path'] }}"
backup-mysqlbinlog-path = "{{ parameter_dict['mysqlbinlog-path'] }}"
backup-mysqldump-path = "{{ parameter_dict['mysqldump-path'] }}"
# Mail configuration
# Alert email sender (default "mrm@localhost")
mail-from = "{{ parameter_dict['mail-from'] }}"
# Alert email SMTP server address, in host:[port] format (default "localhost:25")
mail-smtp-addr = "{{ parameter_dict['mail-smtp-addr'] }}"
mail-smtp-password = "{{ parameter_dict['mail-smtp-password'] }}"
mail-smtp-user = "{{ parameter_dict['mail-smtp-user'] }}"
# Alert email recipients, separated by commas
mail-to = "{{ parameter_dict['mail-to'] }}"
prov-orchestrator = "slapos"
prov-db-tags = "{{ parameter_dict['enabled-tags'] }}"
sysbench-binary-path = "{{ parameter_dict['sysbench-bin'] }}"
# Number of threads to run benchmark (default 4)
sysbench-threads = 4
# Time to run benchmark (default 100)
sysbench-time = 100
sysbench-v1 = true
software/repman/templates/dbjobs.in 0 → 100644
View file @ ce787d53
#!/bin/bash
USER={{ parameter_dict['db-user'] }}
PASSWORD={{ parameter_dict['db-password'] }}
ERROLOG={{ parameter_dict['mysql-dir'] }}/.system/logs/errors.log
SLOWLOG={{ parameter_dict['mysql-dir']}}/.system/logs/sql-slow
BACKUPDIR={{ parameter_dict['mysql-dir'] }}/.system/backup
DATADIR={{ parameter_dict['mysql-dir'] }}/
{% if parameter_dict['use-ipv6'] == True -%}
{% set listen = "TCP6-LISTEN" -%}
{% else -%}
{% set listen = "TCP-LISTEN" -%}
{% endif -%}
export PATH={{ parameter_dict['socat-location'] }}/bin:{{ parameter_dict['mysql-location'] }}/bin:{{ parameter_dict['gzip-location'] }}/bin:$PATH
JOBS=( "xtrabackup" "mariabackup" "error" "slowquery" "zfssnapback" "optimize" "reseedxtrabackup" "reseedmariabackup" "reseedmysqldump" "flashbackxtrabackup" "flashbackmariadbackup" "flashbackmysqldump" "stop" "start")
doneJob()
{
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;UPDATE replication_manager_schema.jobs set end=NOW(), result=LOAD_FILE('{{ parameter_dict['log-dir'] }}/dbjob.out') WHERE id='$ID';" &
}
pauseJob()
{
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "select sleep(6);set sql_log_bin=0;UPDATE replication_manager_schema.jobs set result=LOAD_FILE('{{ parameter_dict['log-dir'] }}/dbjob.out') WHERE id='$ID';" &
}
partialRestore()
{
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;install plugin BLACKHOLE soname 'ha_blackhole.so'"
for dir in $(ls -d $BACKUPDIR/*/ | xargs -n 1 basename | grep -vE 'mysql|performance_schema|replication_manager_schema') ; do
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;drop database IF EXISTS $dir; CREATE DATABASE $dir;"
for file in $(find $BACKUPDIR/$dir/ -name "*.exp" | xargs -n 1 basename | cut -d'.' --complement -f2-) ; do
cat $BACKUPDIR/$dir/$file.frm | sed -e 's/\x06\x00\x49\x6E\x6E\x6F\x44\x42\x00\x00\x00/\x09\x00\x42\x4C\x41\x43\x4B\x48\x4F\x4C\x45/g' > $DATADIR/$dir/mrm_pivo.frm
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;ALTER TABLE $dir.mrm_pivo engine=innodb;RENAME TABLE $dir.mrm_pivo TO $dir.$file; ALTER TABLE $dir.$file DISCARD TABLESPACE;"
mv $BACKUPDIR/$dir/$file.ibd $DATADIR/$dir/$file.ibd
mv $BACKUPDIR/$dir/$file.exp $DATADIR/$dir/$file.exp
mv $BACKUPDIR/$dir/$file.cfg $DATADIR/$dir/$file.cfg
mv $BACKUPDIR/$dir/$file.TRG $DATADIR/$dir/$file.TRG
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;ALTER TABLE $dir.$file IMPORT TABLESPACE"
done
for file in $(find $BACKUPDIR/$dir/ -name "*.MYD" | xargs -n 1 basename | cut -d'.' --complement -f2-) ; do
mv $BACKUPDIR/$dir/$file.* $DATADIR/$dir/
mysql --defaults-file=/etc/mysql/dbjob.cnf -e "set sql_log_bin=0;FLUSH TABLE $dir.$file"
done
for file in $(find $BACKUPDIR/$dir/ -name "*.CSV" | xargs -n 1 basename | cut -d'.' --complement -f2-) ; do
mv $BACKUPDIR/$dir/$file.* $DATADIR/$dir/
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;FLUSH TABLE $dir.$file"
done
done
for file in $(find $BACKUPDIR/mysql/ -name "*.MYD" | xargs -n 1 basename | cut -d'.' --complement -f2-) ; do
mv $BACKUPDIR/mysql/$file.* $DATADIR/mysql/
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;FLUSH TABLE mysql.$file"
done
cat $BACKUPDIR/xtrabackup_info | grep binlog_pos | awk -F, '{ print $3 }' | sed -e 's/GTID of the last change/set sql_log_bin=0;set global gtid_slave_pos=/g' | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e"flush privileges;start slave;"
}
kill -9 $(lsof -t -i:{{ parameter_dict['socat-port'] }} -sTCP:LISTEN)
for job in "${JOBS[@]}"
do
TASK=($(echo "select concat(id,'@',server,':',port) from replication_manager_schema.jobs WHERE task='$job' and done=0 order by task desc limit 1" | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER -N))
ADDRESS=($(echo $TASK | awk -F@ '{ print $2 }'))
ID=($(echo $TASK | awk -F@ '{ print $1 }'))
#purge de past
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e "set sql_log_bin=0;UPDATE replication_manager_schema.jobs set done=1 WHERE done=0 AND task='$job';"
if [ "$ADDRESS" == "" ]; then
echo "No $job needed"
else
echo "Processing $job"
case "$job" in
reseedmysqldump)
echo "Waiting backup." > {{ parameter_dict['log-dir'] }}/dbjob.out
pauseJob
socat -u {{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | gunzip | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER --init-command="reset master;set sql_log_bin=0" > {{ parameter_dict['log-dir'] }}/dbjob.out 2>&1
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e 'start slave;'
;;
flashbackmysqldump)
echo "Waiting backup." > {{ parameter_dict['log-dir'] }}/dbjob.out
pauseJob
socat -u {{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | gunzip | mysql -h{{ parameter_dict['ip'] }} -P{{ parameter_dict['port'] }} -p$PASSWORD -u$USER --init-command="set sql_log_bin=0" > {{ parameter_dict['log-dir'] }}/dbjob.out 2>&1
mysql --defaults-file={{ parameter_dict['dbjob-cnf'] }} -e 'start slave;'
;;
reseedmariabackup)
rm -rf $BACKUPDIR
mkdir $BACKUPDIR
echo "Waiting backup." > {{ parameter_dict['log-dir'] }}/dbjob.out
pauseJob
socat -u {{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | mbstream -x -C $BACKUPDIR
# mbstream -p, --parallel
mariabackup --prepare --export --target-dir=$BACKUPDIR
partialRestore
;;
flashbackmariadbackup)
rm -rf $BACKUPDIR
mkdir $BACKUPDIR
echo "Waiting backup." > {{ parameter_dict['log-dir'] }}/dbjob.out
pauseJob
socat -u {{ listen }}:{{ parameter_dict['socat-port'] }},bind={{ parameter_dict['host'] }},reuseaddr STDOUT | xbstream -x -C $BACKUPDIR
mariabackup --prepare --export --target-dir=$BACKUPDIR
partialRestore
;;
mariabackup)
cd {{ parameter_dict['tmp-dir'] }}
mariadb-backup --innobackupex --defaults-file={{ parameter_dict['dbjob-cnf'] }} --socket='{{ parameter_dict["mysqld-socket"] }}' --no-version-check --user=$USER --password=$PASSWORD --stream=xbstream {{ parameter_dict['tmp-dir'] }}/ | socat -u stdio TCP:$ADDRESS &>{{ parameter_dict['log-dir'] }}/dbjob.out
;;
error)
cat $ERROLOG| socat -u stdio TCP:$ADDRESS &>{{ parameter_dict['log-dir'] }}/dbjob.out
;;
slowquery)
cat $SLOWLOG| socat -u stdio TCP:$ADDRESS &>{{ parameter_dict['log-dir'] }}/dbjob.out
;;
optimize)
mysqlcheck --defaults-file={{ parameter_dict['dbjob-cnf'] }} -o --all-databases --skip-write-binlog &>{{ parameter_dict['log-dir'] }}/dbjob.out
;;
restart)
{{ parameter_dict['restart-script'] }} > {{ parameter_dict['log-dir'] }}/dbjob.out
;;
esac
doneJob
fi
done
software/repman/templates/init_root_wrapper.in 0 → 100644
View file @ ce787d53
#!/bin/bash
run_mysql () {
{{ mysql_bin }} --defaults-file="{{ mysql_conf }}" \
--protocol=socket -uroot -hlocalhost $@
}
if [ ! -f "{{ init_password_done }}" ]; then
for i in {30..0}; do
if echo 'SELECT 1' | run_mysql &> /dev/null; then
break
fi
echo 'MySQL init process in progress...'
sleep 1
done
if [ "$i" = 0 ]; then
echo >&2 'MySQL init process failed.'
exit 1
fi
echo "Setting mariabdb root password...";
run_mysql < {{ init_root_sql }} && touch {{ init_password_done }} || exit 1;
echo "done"
fi
# Run mariadb_upgrade when replication is bootstrapped will break replication topology.
# skip when already upgraded until we have a good solution.
if [ ! -f "{{ upgrade_done }}" ]; then
{{ mysql_update }}
if [ $? -eq 0 ]; then
touch {{ upgrade_done }};
fi
fi
\ No newline at end of file
software/repman/templates/mariadb_init_root.sql.in 0 → 100644
View file @ ce787d53
-- What's done in this file shouldn't be replicated
-- or products like mysql-fabric won't work
SET @@SESSION.SQL_LOG_BIN=0;
CREATE USER '{{ parameter_dict["root-user"] }}'@'localhost' IDENTIFIED BY '{{ parameter_dict["password"] }}' ;
GRANT ALL ON *.* TO '{{ parameter_dict["root-user"] }}'@'localhost' WITH GRANT OPTION ;
CREATE USER '{{ parameter_dict["root-user"] }}'@'%' IDENTIFIED BY '{{ parameter_dict["password"] }}' ;
GRANT ALL ON *.* TO '{{ parameter_dict["root-user"] }}'@'%' WITH GRANT OPTION ;
CREATE USER '{{ parameter_dict["root-user"] }}'@'::' IDENTIFIED BY '{{ parameter_dict["password"] }}' ;
GRANT ALL ON *.* TO '{{ parameter_dict["root-user"] }}'@'::' WITH GRANT OPTION ;
CREATE USER '{{ parameter_dict["heartbeat-user"] }}'@'localhost' IDENTIFIED BY '{{ parameter_dict["password"] }}' ;
GRANT ALL ON *.* TO '{{ parameter_dict["heartbeat-user"] }}'@'localhost' WITH GRANT OPTION ;
CREATE USER '{{ parameter_dict["heartbeat-user"] }}'@'%' IDENTIFIED BY '{{ parameter_dict["password"] }}' ;
GRANT ALL ON *.* TO '{{ parameter_dict["heartbeat-user"] }}'@'%' WITH GRANT OPTION ;
DROP DATABASE IF EXISTS test ;
FLUSH PRIVILEGES ;
\ No newline at end of file
software/repman/templates/mariadb_initial_setup.sql.in 0 → 100644
View file @ ce787d53
SET @@SESSION.SQL_LOG_BIN=0;
USE mysql;
{% set mroonga = parameter_dict.get('mroonga', 'ha_mroonga.so') -%}
{% if mroonga %}
SOURCE {{ parameter_dict['mroonga-mariadb-install-sql'] }};
{% endif %}
DROP FUNCTION IF EXISTS sphinx_snippets;
#CREATE FUNCTION sphinx_snippets RETURNS STRING SONAME 'ha_sphinx.so';
{% macro database(name, user, password) -%}
CREATE DATABASE IF NOT EXISTS `{{ name }}`;
{% if user -%}
GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@`%` IDENTIFIED BY '{{ password }}';
GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@localhost IDENTIFIED BY '{{ password }}';
GRANT ALL PRIVILEGES ON `{{ name }}`.* TO `{{ user }}`@'::' IDENTIFIED BY '{{ password }}';
{%- endif %}
{% endmacro -%}
{% for entry in parameter_dict['database-list'] -%}
{{ database(entry['name'], entry.get('user'), entry.get('password')) }}
{% endfor -%}
\ No newline at end of file
software/repman/templates/my.cnf.in 0 → 100644
View file @ ce787d53
{% set socket = parameter_dict['socket'] -%}
# ERP5 buildout my.cnf template based on my-huge.cnf shipped with mysql
# The MySQL server
[mysqld]
# ERP5 by default requires InnoDB storage. MySQL by default fallbacks to using
# different engine, like MyISAM. Such behaviour generates problems only, when
# tables requested as InnoDB are silently created with MyISAM engine.
#
# Loud fail is really required in such case.
# Already present in REPMAN
#sql_mode="NO_ENGINE_SUBSTITUTION"
socket = {{ socket }}
datadir = {{ parameter_dict['data-directory'] }}
pid_file = {{ parameter_dict['pid-file'] }}
{% set innodb_buffer_pool_size = parameter_dict['innodb-buffer-pool-size'] -%}
{% if innodb_buffer_pool_size %}innodb_buffer_pool_size = {{ innodb_buffer_pool_size }}{% endif %}
{% set innodb_buffer_pool_instances = parameter_dict['innodb-buffer-pool-instances'] -%}
{% if innodb_buffer_pool_instances %}innodb_buffer_pool_instances = {{ innodb_buffer_pool_instances }}{% endif %}
{% set innodb_log_file_size = parameter_dict['innodb-log-file-size'] -%}
{% if innodb_log_file_size %} innodb_log_file_size = {{ innodb_log_file_size }}{% endif %}
{% set innodb_log_buffer_size = parameter_dict['innodb-log-buffer-size'] -%}
{% if innodb_log_buffer_size %} innodb_log_buffer_size = {{ innodb_log_buffer_size }}{% endif %}
# very important to allow parallel indexing
# Note: this is compatible with binlog-based incremental backups, because ERP5
# doesn't use "insert ... select" (in any number of queries) pattern.
# innodb_locks_unsafe_for_binlog = 1
#plugin_load = ha_mroonga
plugin-dir = {{ parameter_dict['plugin-directory'] }}
{% if 'ssl-key' in parameter_dict -%}
ssl
ssl-cert = {{ parameter_dict['ssl-crt'] }}
ssl-key = {{ parameter_dict['ssl-key'] }}
{% if 'ssl-ca-crt' in parameter_dict -%}
ssl-ca = {{ parameter_dict['ssl-ca-crt'] }}
{%- endif %}
{% if 'ssl-crl' in parameter_dict -%}
ssl-crl = {{ parameter_dict['ssl-crl'] }}
{%- endif %}
{% if 'ssl-cipher' in parameter_dict -%}
ssl-cipher = {{ parameter_dict['ssl-cipher'] }}
{%- endif %}
{%- endif %}
# Some dangerous settings you may want to uncomment temporarily
# if you only want performance or less disk access.
{% set x = '' if parameter_dict['relaxed-writes'] else '#' -%}
{{x}}innodb_flush_log_at_trx_commit = 0
{{x}}innodb_flush_method = nosync
{{x}}innodb_doublewrite = 0
{{x}}sync_frm = 0
# skip_character_set_client_handshake
[client]
socket = {{ socket }}
user = root
[mysql]
no_auto_rehash
[mysqlhotcopy]
interactive_timeout
[mysqldump]
max_allowed_packet = 128M
software/repman/templates/mysqld-need-start.sh.in 0 → 100644
View file @ ce787d53
#!{{ bash_bin }}
curl () {
{{ curl_bin }} -k --silent -H "Accept: application/json" "$@"
}
# TOKEN=$(curl -s -X POST --data '{"username":"{{ username }}","password":"XXXXX"}' {{ repman_url }}/api/login | {{ jq_bin }} -r '.token')
# Checking if mariadb start is needed
#CODE=$(curl -H "Authorization: Bearer ${TOKEN}" -o /dev/null -w "%{http_code}" {{ repman_url }}/api/clusters/{{ cluster }}/servers/{{ db_host }}/{{ db_port }}/need-start)
CODE=$(curl -o /dev/null -w "%{http_code}" {{ repman_url }}/api/clusters/{{ cluster }}/servers/{{ db_host }}/{{ db_port }}/need-start)
if [ $CODE -eq 200 ]; then
echo "$CODE: Updating mysql configuration..."
# update mysql configuration
{{ update_config }}
echo "$CODE: Starting mariadb service..."
# print current status, can be useful for debug...
{{ mariadb_controller }} status mariadb
{{ mariadb_controller }} start mariadb
sleep 5
# check again if the service is still up...
{{ mariadb_controller }} status mariadb
fi
software/repman/templates/nginx.conf.in 0 → 100644
View file @ ce787d53
pid {{ parameter_dict['pid-file'] }};
error_log {{ parameter_dict['error-log'] }};
daemon off;
events {
worker_connections 1024;
accept_mutex off;
}
http {
default_type application/octet-stream;
access_log {{ parameter_dict['access-log'] }} combined;
client_max_body_size 10M;
map $http_upgrade $connection_upgrade {
default upgrade;
'' close;
}
server {
listen [{{ parameter_dict['ipv6'] }}]:{{ parameter_dict['ssl-port'] }} ssl;
server_name _;
ssl_certificate {{ parameter_dict['ssl-certificate'] }};
ssl_certificate_key {{ parameter_dict['ssl-key'] }};
ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
ssl_ciphers ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:HIGH:!aNULL:!MD5;
ssl_prefer_server_ciphers on;
keepalive_timeout 90s;
location / {
proxy_redirect off;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Host $http_host;
proxy_connect_timeout 200;
proxy_pass {{ parameter_dict['repman-secure-url'] }};
}
}
server {
listen [{{ parameter_dict['ipv6'] }}]:{{ parameter_dict['port'] }};
server_name _;
location / {
proxy_redirect off;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Host $http_host;
proxy_connect_timeout 200;
proxy_pass {{ parameter_dict['repman-url'] }};
}
}
}
software/repman/templates/proxy-need-start-stop.sh.in 0 → 100644
View file @ ce787d53
#!{{ bash_bin }}
curl () {
{{ curl_bin }} -k --silent "$@"
}
check_start_cluster () {
NAME=$1
HOST=$2
PORT=$3
CODE=$(curl -o /dev/null -w "%{http_code}" {{ repman_url }}/api/clusters/$NAME/servers/$HOST/$PORT/need-start)
if [ $CODE -eq 200 ]; then
echo "$CODE: Starting proxysql $HOST:$PORT..."
{{ proxysql_controller }} start proxysql-$NAME
sleep 1
# check again if the service is still up...
{{ proxysql_controller }} status proxysql-$NAME
fi
}
check_stop_cluster () {
NAME=$1
HOST=$2
PORT=$3
CODE=$(curl -o /dev/null -w "%{http_code}" {{ repman_url }}/api/clusters/$NAME/servers/$HOST/$PORT/need-stop)
if [ $CODE -eq 200 ]; then
echo "$CODE: updating proxysql config $HOST:$PORT..."
{{ get_proxy_config }} $NAME $HOST $PORT
echo "$CODE: Stoping proxysql $HOST:$PORT..."
{{ proxysql_controller }} stop proxysql-$NAME
sleep 1
# check again if the service is stopped...
{{ proxysql_controller }} status proxysql-$NAME
fi
}
{% for cluster_dict in cluster_list -%}
check_start_cluster {{ cluster_dict['name'] }} {{ cluster_dict['host'] }} {{ cluster_dict['port'] }}
check_stop_cluster {{ cluster_dict['name'] }} {{ cluster_dict['host'] }} {{ cluster_dict['port'] }}
{% endfor -%}
software/repman/templates/repman-manager.sh.in 0 → 100644
View file @ ce787d53
#!{{ bash_bin }}
#set -e
curl () {
{{ curl_bin }} -k --silent -H "Accept: application/json" "$@"
}
get_token () {
curl -s -X POST --data '{"username":"{{ username }}","password":"{{ password}}"}' {{ secure_url }}/api/login
}
wait_database () {
NAME=$1
for retry in {1..50}; do
echo ">> Wait until $NAME databases are ready...";
CODE=$(curl -H "Authorization: Bearer ${TOKEN}" -o /dev/null -w "%{http_code}" {{ secure_url }}/api/clusters/$NAME/actions/waitdatabases);
if [ $CODE -eq 504 ]; then
# We have a timeout try again
echo ">> [$retry] Timeout on {{ secure_url }}/api/clusters/$NAME/actions/waitdatabases, trying again...";
else
if [ $CODE -eq 200 ]; then
break;
else
if [ $CODE -eq 401 ]; then
# try again with new token
TOKEN=$(get_token | {{ jq_bin }} -r '.token')
fi
fi
echo ">> [$retry] waitdatabases returned code $CODE...";
fi
sleep 30
done
}
activate_proxy () {
NAME=$1
URL="{{ secure_url }}/api/clusters/$NAME/settings/actions/switch/database-hearbeat"
echo ">> Calling $URL...";
CODE=$(curl -H "Authorization: Bearer ${TOKEN}" -o /dev/null -w "%{http_code}" $URL)
if [ $CODE -eq 200 ]; then
return 0;
else
echo ">> ERROR: failed to activate proxy: $URL returned code $CODE"
return 1;
fi
}
TOKEN=$(get_token | {{ jq_bin }} -r '.token')
# Always reload cluster configuration to apply recent changes
{% for name in cluster_name_list -%}
# reload {{ name }} settings
echo "Reloading settings for {{ name }}..."
curl -H "Authorization: Bearer ${TOKEN}" \
{{ secure_url }}/api/clusters/{{ name }}/settings/actions/reload
# Start Replication on {{ name }}
if [ ! -f "{{ parameter_dict['bootstrap'] }}/{{ name }}_bootstrapped" ]; then
wait_database {{ name }}
echo "Bootstrap replication on {{ name }}..."
TOKEN=$(get_token | {{ jq_bin }} -r '.token')
curl -H "Authorization: Bearer ${TOKEN}" \
{{ secure_url }}/api/clusters/{{ name }}/actions/replication/cleanup
CODE=$(curl -H "Authorization: Bearer ${TOKEN}" -o /dev/null -w "%{http_code}" {{ secure_url }}/api/clusters/{{ name }}/actions/replication/bootstrap/master-slave)
SUCCESS=0
if [ $CODE -eq 200 ]; then
activate_proxy {{ name }}
if [ $? -eq 0 ]; then
# Mark boostrap done!
echo "Cluster {{ name }} replication bootstrapped"
echo "DO NOT REMOVE THIS FILE" > {{ parameter_dict['bootstrap'] }}/{{ name }}_bootstrapped
fi
else
echo "ERROR: Failed to bootstrap cluster {{ name }}... http_code $CODE"
fi
fi
{% endfor %}
\ No newline at end of file
software/repman/test/README.md 0 → 100644
View file @ ce787d53
Tests for Replication Manager software release
software/repman/test/setup.py 0 → 100644
View file @ ce787d53
##############################################################################
#
# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from setuptools import setup, find_packages
version = '0.0.1.dev0'
name = 'slapos.test.repman'
long_description = open("README.md").read()
setup(
name=name,
version=version,
description="Test for SlapOS' Replication Manager",
long_description=long_description,
long_description_content_type='text/markdown',
maintainer="Nexedi",
maintainer_email="info@nexedi.com",
url="https://lab.nexedi.com/nexedi/slapos",
packages=find_packages(),
install_requires=[
'slapos.core',
'slapos.libnetworkcache',
'erp5.util',
'supervisor',
'pexpect',
'requests',
],
zip_safe=True,
test_suite='test',
)
software/repman/test/test.py 0 → 100644
View file @ ce787d53
##############################################################################
#
# Copyright (c) 2019 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from __future__ import unicode_literals
import os
import textwrap
import logging
import tempfile
import time
from six.moves.urllib.parse import urlparse, urljoin
import pexpect
import requests
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
class TestRepman(SlapOSInstanceTestCase):
__partition_reference__ = 'R' # solve path too long for postgresql and unicorn
@classmethod
def getInstanceSoftwareType(cls):
return 'default'
def setUp(self):
self.backend_url = self.computer_partition.getConnectionParameterDict(
)['backend-url']
def test_http_get(self):
resp = requests.get(self.backend_url, verify=False)
self.assertTrue(
resp.status_code in [requests.codes.ok, requests.codes.found])
software/slapos-sr-testing/software.cfg
View file @ ce787d53
...@@ -148,6 +148,11 @@ setup = ${slapos-repository:location}/software/cloudooo/test/ ...@@ -148,6 +148,11 @@ setup = ${slapos-repository:location}/software/cloudooo/test/
egg = slapos.test.dream egg = slapos.test.dream
setup = ${slapos-repository:location}/software/dream/test/ setup = ${slapos-repository:location}/software/dream/test/
[slapos.test.repman-setup]
<= setup-develop-egg
egg = slapos.test.repman
setup = ${slapos-repository:location}/software/repman/test/
[slapos.core-repository] [slapos.core-repository]
<= git-clone-repository <= git-clone-repository
repository = https://lab.nexedi.com/nexedi/slapos.core.git repository = https://lab.nexedi.com/nexedi/slapos.core.git
...@@ -193,6 +198,7 @@ eggs = ...@@ -193,6 +198,7 @@ eggs =
${slapos.test.cloudooo-setup:egg} ${slapos.test.cloudooo-setup:egg}
${slapos.test.dream-setup:egg} ${slapos.test.dream-setup:egg}
${slapos.test.metabase-setup:egg} ${slapos.test.metabase-setup:egg}
${slapos.test.repman-setup:egg}
${backports.lzma:egg} ${backports.lzma:egg}
entry-points = entry-points =
runTestSuite=erp5.util.testsuite:runTestSuite runTestSuite=erp5.util.testsuite:runTestSuite
...@@ -259,6 +265,7 @@ extra = ...@@ -259,6 +265,7 @@ extra =
${slapos.test.gitlab-setup:setup} ${slapos.test.gitlab-setup:setup}
${slapos.test.cloudooo-setup:setup} ${slapos.test.cloudooo-setup:setup}
${slapos.test.dream-setup:setup} ${slapos.test.dream-setup:setup}
${slapos.test.repman-setup:setup}
[versions] [versions]
# slapos.core is used from the clone always # slapos.core is used from the clone always
......
stack/caucase/buildout.cfg
View file @ ce787d53
...@@ -35,6 +35,6 @@ mode = 0644 ...@@ -35,6 +35,6 @@ mode = 0644
depends = ${caucase-jinja2-library-eggs:eggs} depends = ${caucase-jinja2-library-eggs:eggs}
[versions] [versions]
caucase = 0.9.5 caucase = 0.9.7
pem = 17.1.0 pem = 17.1.0
PyJWT = 1.7.1 PyJWT = 1.7.1
stack/erp5/buildout.cfg
View file @ ce787d53
...@@ -610,7 +610,7 @@ zope.app.testing = 3.8.1 ...@@ -610,7 +610,7 @@ zope.app.testing = 3.8.1
# Pinned versions # Pinned versions
APacheDEX = 1.6.2 APacheDEX = 1.6.2
Pillow = 5.2.0 Pillow = 6.2.2
Products.CMFActionIcons = 2.1.3 Products.CMFActionIcons = 2.1.3
Products.DCWorkflowGraph = 0.4.1 Products.DCWorkflowGraph = 0.4.1
# Products.ExternalEditor 2.0.0's dtml is not based on Zope2 OFS's one. # Products.ExternalEditor 2.0.0's dtml is not based on Zope2 OFS's one.
...@@ -706,7 +706,7 @@ fpconst = 0.7.2 ...@@ -706,7 +706,7 @@ fpconst = 0.7.2
graphviz = 0.5.2 graphviz = 0.5.2
# Required by: # Required by:
# Pillow==4.0.0 # Pillow==6.2.2
olefile = 0.44 olefile = 0.44
# Required by: # Required by:
......
stack/slapos.cfg
View file @ ce787d53
...@@ -95,6 +95,7 @@ eggs = ...@@ -95,6 +95,7 @@ eggs =
${python-cachecontrol:egg} ${python-cachecontrol:egg}
${python-cliff:egg} ${python-cliff:egg}
${python-cryptography:egg} ${python-cryptography:egg}
${jsonschema:egg}
pyOpenSSL pyOpenSSL
slapos.cookbook slapos.cookbook
...@@ -107,6 +108,11 @@ eggs = ...@@ -107,6 +108,11 @@ eggs =
${python-cryptography:egg} ${python-cryptography:egg}
slapos.toolbox slapos.toolbox
[jsonschema]
recipe = zc.recipe.egg:custom
egg = ${:_buildout_section_name_}
setup-eggs = setuptools_scm
# Install a slapos command with networkcache enabled in ${buildout:bin-directory} # Install a slapos command with networkcache enabled in ${buildout:bin-directory}
[slapos-command] [slapos-command]
recipe = zc.recipe.egg recipe = zc.recipe.egg
...@@ -230,6 +236,10 @@ attrs = 18.2.0 ...@@ -230,6 +236,10 @@ attrs = 18.2.0
# jsonschema==3.0.0a3 # jsonschema==3.0.0a3
pyrsistent = 0.14.5 pyrsistent = 0.14.5
# Required by:
# jsonschema==3.0.2
setuptools-scm = 3.5.0
# Required by: # Required by:
# cryptography==1.8.1 # cryptography==1.8.1
ipaddress = 1.0.18 ipaddress = 1.0.18
......
stack/supervisord/README.rst 0 → 100644
View file @ ce787d53
Supervisord process manager
How to use
==========
Supervisord stack provides a library which can be called in your instance slapos. This stack can be used to run sub services in a partition.
To use:
* extend ``stack/supervisord/buildout.cfg`` in your software.cfg file.
* provide ``supervisord-library:target`` and ``supervisord-conf:target`` to your instance template which require to use supervisord controller.
* add ``{% import "supervisord" as supervisord with context %}`` to instance template which call supervisord library. See example below:
**software.cfg**
::
[template-instance]
recipe = slapos.recipe.template:jinja2
context =
key buildout_bin_directory buildout:bin-directory
key supervisord supervisord-library:target
key supervisord_conf supervisord-conf:target
**instance.cfg.in**
::
[template-custom-instance.cfg]
recipe = slapos.recipe.template:jinja2
supervisord-lib = {{ supervisord }}
import-list =
file supervisord :supervisord-lib
context =
raw buildout_bin_directory {{ buildout_bin_directory }}
raw supervisord_conf {{ supervisord_conf }}
**custom-instance.cfg**
::
{% import "supervisord" as supervisord with context %}
{{ supervisord.supervisord("custom-controller", buildout_bin_directory, supervisord_conf, use_service_hash=False) }}
# add program to service controller
{% set program_dict = {"name": "mariadb", "command": "${mariadb-service:wrapper}",
"stopwaitsecs": 300, "environment": []} %}
{{ supervisord.supervisord_program("mariadb", program_dict) }}
...
[buildout]
parts =
...
supervisord-custom-controller
supervisord-mariadb
Supervisord inside partition
============================
Check supervisord controlled services status:
::
$ instance/slappartXX/bin/custom-controller status
mariadb RUNNING pid 5511, uptime 6:04:54
`supervisord_program` parameters and defaults:
.. code-block:: python
program_dict = {
"name": "NAME",
"command": "WRAPPER_PATH",
"stopwaitsecs": 60,
"environment": ['PATH="/usr/bin/:/partition/bin/:$PATH"', 'MAKEFLAGS="-j2"'],
"autostart": True,
"autorestart": False,
"startsecs": 0,
"startretries": 0,
"stopsignal": "TERM",
"stdout_logfile": "NONE",
"stderr_logfile": "NONE"
}
stack/supervisord/buildout.cfg 0 → 100644
View file @ ce787d53
[buildout]
extends =
../slapos.cfg
buildout.hash.cfg
parts =
supervisord-conf
supervisord-library
[supervisord-download-base]
recipe = slapos.recipe.build:download
mode = 0644
url = ${:_profile_base_location_}/${:_update_hash_filename_}
[supervisord-eggs]
recipe = zc.recipe.egg
eggs =
${slapos-cookbook:eggs}
supervisor
scripts =
supervisord
supervisorctl
[supervisord-library]
<= supervisord-download-base
filename = supervisord.jinja2.in
depends = ${supervisord-eggs:recipe}
[supervisord-conf]
<= supervisord-download-base
filename = supervisord.conf.in
stack/supervisord/buildout.hash.cfg 0 → 100644
View file @ ce787d53
# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
# The only allowed lines here are (regexes):
# - "^#" comments, copied verbatim
# - "^[" section beginings, copied verbatim
# - lines containing an "=" sign which must fit in the following categorie.
# - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
# Copied verbatim.
# - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
# by the re-generation script.
# Re-generated.
# - other lines are copied verbatim
# Substitution (${...:...}), extension ([buildout] extends = ...) and
# section inheritance (< = ...) are NOT supported (but you should really
# not need these here).
[supervisord-library]
_update_hash_filename_ = supervisord.jinja2.in
md5sum = 163c9f60e4ad3842162cbb11d771b7b8
[supervisord-conf]
_update_hash_filename_ = supervisord.conf.in
md5sum = d624f65151233493c6dbdafa83ae8cbd
stack/supervisord/supervisord.conf.in 0 → 100644
View file @ ce787d53
[unix_http_server]
file = {{ parameter_dict['socket-path'] }}
chmod=0700
[include]
files = {{ parameter_dict['include-dir'] }}/*.conf
[supervisorctl]
serverurl = unix://{{ parameter_dict['socket-path'] }}
[supervisord]
loglevel = {{ parameter_dict['log-level'] }}
logfile_maxbytes = 2MB
nodaemon = false
logfile-backups = 3
logfile = {{ parameter_dict['log-file'] }}
pidfile = {{ parameter_dict['pid-file'] }}
[rpcinterface:supervisor]
supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
stack/supervisord/supervisord.jinja2.in 0 → 100644
View file @ ce787d53
{% macro supervisord(
name,
buildout_bin_directory,
supervisord_conf,
use_service_hash=False
) -%}
[controller-directory]
recipe = slapos.cookbook:mkdirectory
etc = ${buildout:directory}/etc
var = ${buildout:directory}/var
log = ${:var}/log
run = ${:var}/run
supervisord = ${:etc}/supervisord-{{ name }}.conf.d
[controller-parameters]
socket-path = ${controller-directory:run}/{{ name }}.socket
include-dir = ${controller-directory:supervisord}
log-file = ${controller-directory:log}/supervisord-{{ name }}.log
log-level = info
pid-file = ${controller-directory:run}/supervisord-{{ name }}.pid
[supervisord-controller-conf]
recipe = slapos.recipe.template:jinja2
template = {{ supervisord_conf }}
context =
section parameter_dict controller-parameters
rendered = ${controller-directory:etc}/supervisord-{{ name }}.conf
[supervisord-{{ name }}]
recipe = slapos.cookbook:wrapper
command-line = {{ buildout_bin_directory }}/supervisord -c ${supervisord-controller-conf:rendered} --nodaemon
wrapper-path = ${directory:services}/supervisord-{{ name }}
{% if use_service_hash -%}
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
{% endif -%}
depends =
{{ '${' ~ name ~ '-bin:recipe}' }}
[{{ name }}-bin]
recipe = slapos.cookbook:wrapper
command-line = {{ buildout_bin_directory }}/supervisorctl -c ${supervisord-controller-conf:rendered}
wrapper-path = ${directory:bin}/{{ name }}
{%- endmacro %}
{% macro supervisord_program(
name,
parameter_dict
) -%}
[supervisord-{{ name }}]
recipe = slapos.recipe.template:jinja2
template = inline:[program:{{ parameter_dict['name'] }}]
directory = ${buildout:directory}
command = {{ parameter_dict['command'] }}
process_name = {{ parameter_dict['name'] }}
autostart = {{ parameter_dict.get('autostart', True) }}
autorestart = {{ parameter_dict.get('autorestart', False) }}
startsecs = {{ parameter_dict.get('startsecs', 0) }}
startretries = {{ parameter_dict.get('startretries', 0) }}
exitcodes = {{ parameter_dict.get('exitcodes', 0) }}
stopsignal = {{ parameter_dict.get('stopsignal', 'TERM') }}
stopwaitsecs = {{ parameter_dict.get('stopwaitsecs', 60) }}
serverurl=AUTO
redirect_stderr=true
stdout_logfile = {{ parameter_dict.get('stdout_logfile', 'NONE') }}
stdout_logfile_maxbytes = 1000KB
stdout_logfile_backups = 1
stderr_logfile = {{ parameter_dict.get('stderr_logfile', 'NONE') }}
stderr_logfile_maxbytes = 1000KB
stderr_logfile_backups = 1
environment = {{ parameter_dict['environment'] | join(',') }}
rendered = ${controller-directory:supervisord}/{{ name }}.conf
{%- endmacro %}
  • Jérome Perrin @jerome ·
    Owner

    In this release KVM and gitlab are broken ( for what seems to be a buildout bug, because for gitlab a workaround like jerome/slapos@5806ed5c seems to fix ).

    @rafael @tomo How should we do when we need to release but master is broken ?

    PS: is anybody working on repairing the state on master ? += and extends have always been buggy in buildout, maybe we should apply this gitlab patch and do same in kvm ?

  • Rafael Monnerat @rafael ·
    Owner

    How should we do when we need to release but master is broken ?

    It is a bit complex but couldn't it get fixed by who broken it or make a workarround? KVM needs to be stable for a project so, in any case, it should be fixed somehow.

  • Thomas Gambier 🚴🏼 @tomo ·
    Owner

    How should we do when we need to release but master is broken ?

    It's OK to merge on 1.0 branch even if all software are not working. The purpose of merging in 1.0 is to be able to tag and generate a SR in slapos master. We won't generate a SR in slapos master for broken SR for sure.

    And of course, master should be fixed in the meantime.

    PS: is anybody working on repairing the state on master ? += and extends have always been buggy in buildout, maybe we should apply this gitlab patch and do same in kvm ?

    You can have a look at https://www.erp5.com/group_section/forum/buildout--extends-and---cheat-sheet-O0wP6uGjM1

Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7