*: compatibility with DateTime >= 3 🚧

drop the patch in DateTime._parse_args which cause DateTime to
support timezone naive dates

adjust __setstate__ to pickle the timezone naive flag

adjust code relying on DateTime private attributes

adjust tests for new DateTime.__eq__ behavior (instances with
different timezones are different)

more testing, especially for various __setstate__ versions

introduce a context manager to change timezone in tests.

bt5/erp5_core_test/TestTemplateItem/portal_components/test.erp5.testDateUtils.py

@@ -27,11 +27,13 @@
 #
 ##############################################################################
 
+import os
 import unittest
 
+import zodbpickle.fastpickle as pickle
 from DateTime import DateTime
 from erp5.component.module.DateUtils import addToDate, getIntervalListBetweenDates, \
-    atTheEndOfPeriod, getClosestDate
+    atTheEndOfPeriod, getClosestDate, timeZoneContext
 from Products.ERP5Type.tests.ERP5TypeTestCase import ERP5TypeTestCase
 
 class TestDateUtils(unittest.TestCase):
@@ -199,8 +201,126 @@ class TestPinDateTime(ERP5TypeTestCase):
     self.assertGreaterEqual(DateTime(), actual_begin_date)
 
 
+class TestTimeZoneContext(ERP5TypeTestCase):
+  def afterSetUp(self):
+    self.reference_date_in_utc = DateTime('2001/02/03 00:00:00 UTC')
+    self.actual_timezone = DateTime().timezone()
+    self.actual_environ_tz = os.environ.get('TZ')
+
+  def test_timezone_context_UTC(self):
+    with timeZoneContext('UTC'):
+      self.assertEqual(DateTime().timezone(), 'UTC')
+      self.assertEqual(
+        DateTime(2001, 2, 3).toZone('UTC'), self.reference_date_in_utc)
+    self.assertEqual(DateTime().timezone(), self.actual_timezone)
+    self.assertEqual(os.environ.get('TZ'), self.actual_environ_tz)
+
+  def test_timezone_context_with_dst(self):
+    with timeZoneContext('Europe/Paris'):
+      self.assertEqual(DateTime(2021, 2, 1).timezone(), 'CET')
+      self.assertEqual(DateTime(2021, 7, 1).timezone(), 'CEST')
+      self.assertEqual(
+        DateTime(2001, 2, 3, 1, 0, 0).toZone('UTC'),
+        self.reference_date_in_utc)
+    self.assertEqual(DateTime().timezone(), self.actual_timezone)
+    self.assertEqual(os.environ.get('TZ'), self.actual_environ_tz)
+
+  def test_timezone_context_without_dst(self):
+    with timeZoneContext('Asia/Tokyo'):
+      self.assertEqual(DateTime().timezone(), 'JST')
+      self.assertEqual(
+        DateTime(2001, 2, 3, 9, 0, 0).toZone('UTC'), self.reference_date_in_utc)
+    self.assertEqual(DateTime().timezone(), self.actual_timezone)
+    self.assertEqual(os.environ.get('TZ'), self.actual_environ_tz)
+
+  def test_timezone_abbreviation(self):
+    with timeZoneContext('GMT-7'):
+      self.assertEqual(DateTime(2021, 2, 1).timezone(), 'GMT-7')
+      self.assertEqual(DateTime(2021, 7, 1).timezone(), 'GMT-7')
+      self.assertEqual(
+        DateTime(2001, 2, 2, 17, 0, 0).toZone('UTC'), self.reference_date_in_utc)
+    self.assertEqual(DateTime().timezone(), self.actual_timezone)
+    self.assertEqual(os.environ.get('TZ'), self.actual_environ_tz)
+
+
+class TestTimePatch(ERP5TypeTestCase):
+  """Tests for monkey patches in Products.ERP5Type.patches.DateTimePatch
+  """
+  def _test_pickle(self, dt, data):
+    """Assert pickle `data` when loaded is equal to DateTime `dt`
+    """
+    new = pickle.loads(data)
+    if hasattr(DateTime, '__slots__'):
+      for key in DateTime.__slots__:
+        self.assertEqual(getattr(dt, key), getattr(new, key))
+    else:
+      # BBB DateTime 2
+      self.assertEqual(dt.__dict__, new.__dict__)
+
+  # ERP5 custom pickle supports timezone naive DateTimes
+  def test_pickle_timezone_naive(self):
+    dt = DateTime('2001/02/03 04:05:06')
+    self.assertTrue(dt.timezoneNaive())
+    data = b"(NiDateTime.DateTime\nDateTime\np1\n(F981173106\n(S'UTC'\np2\nI01\nttb."
+    self._test_pickle(dt, data)
+    self.assertTrue(pickle.loads(data).timezoneNaive())
+
+  # pickles from ERP5 around commit fcaa5dddbd (Zelenium: update html2canvas to version 1.4.1, 2022-04-18)
+  def test_pickle_europe_paris(self):
+    dt = DateTime('2001/02/03 04:05:06 Europe/Paris')
+    data = b'(cDateTime.DateTime\nDateTime\nq\x01Noq\x02(GA\xcd=\xba\xb1\x00\x00\x00U\x0cEurope/Parisq\x03tb.'
+    self._test_pickle(dt, data)
+
+  def test_pickle_UTC(self):
+    dt = DateTime('2001/02/03 04:05:06 UTC')
+    data = b'(cDateTime.DateTime\nDateTime\nq\x01Noq\x02(GA\xcd=\xc1\xb9\x00\x00\x00U\x03UTCq\x03tb.'
+    self._test_pickle(dt, data)
+
+  # "r15569" was an old patch to DateTime.__getstate__ that we keep comptatibility with.
+  # It was a svn commit that was convert to git commit 7b89b86838 (Tweak DateTime pickle
+  # representation to avoid using 370 bytes per DateTime, but ~80 bytes instead.
+  # Retain backward compatibility with regular DateTime default serialisation., 2007-08-08)
+  def test_pickle_europe_paris_r15569(self):
+    dt = DateTime('2001/02/03 04:05:06 Europe/Paris')
+    data = b'(cDateTime.DateTime\nDateTime\nq\x01Noq\x02}q\x03U\x03strq\x04U 2001/02/03 04:05:06 Europe/Parissb.'
+    self._test_pickle(dt, data)
+
+  def test_pickle_UTC_r15569(self):
+    dt = DateTime('2001/02/03 04:05:06 UTC')
+    data = b'(cDateTime.DateTime\nDateTime\nq\x01Noq\x02}q\x03U\x03strq\x04U\x172001/02/03 04:05:06 UTCsb.'
+    self._test_pickle(dt, data)
+
+  def test_pickle_protocol_3(self):
+    dt = DateTime()
+    data = pickle.dumps(dt, 3)
+    self._test_pickle(dt, data)
+
+  def test_pickle_dumps_loads(self):
+    for i in (
+      '2007/01/02 12:34:56.789',
+      '2007/01/02 12:34:56.789 GMT+0200',
+      '2007/01/02 12:34:56.789 JST',
+      '2007/01/02 12:34:56.789 +0300',
+      '2007/01/02 12:34:56.789 +0430',
+      '2007/01/02 12:34:56.789 +1237',
+    ):
+      dt = DateTime(i)
+      self._test_pickle(dt, pickle.dumps(dt, 1))
+
+
 def test_suite():
   suite = unittest.TestSuite()
   suite.addTest(unittest.makeSuite(TestDateUtils))
   suite.addTest(unittest.makeSuite(TestPinDateTime))
+  suite.addTest(unittest.makeSuite(TestTimeZoneContext))
+  suite.addTest(unittest.makeSuite(TestTimePatch))
+
+  # also run original tests from DateTime module
+  # pylint:disable=no-name-in-module
+  try:
+    import DateTime.tests.testDateTime as test_datetime
+  except ImportError:
+    from DateTime.tests import test_datetime
+  # pylint:enable=no-name-in-module
+  suite.addTest(test_datetime.test_suite())
   return suite

bt5/erp5_web_renderjs_ui_test/ExtensionTemplateItem/portal_components/extension.erp5.WebRenderJSTestUtils.py

-import os, time
-from DateTime import DateTime
+from erp5.component.module.DateUtils import timeZoneContext
+
+current_timezone_contexts = []
 
 def setTimezone(timezone):
-  # timezone must be for example GMT-7
-  os.environ['TZ'] = timezone
-  time.tzset()
-  DateTime._isDST = False
-  DateTime._localzone = DateTime._localzone0 = DateTime._localzone1 = timezone
+  """Change the default timezone to `timezone`.
+  """
+  if current_timezone_contexts:
+    resetTimeZone()
+
+  tzc = timeZoneContext(timezone)
+  tzc.__enter__()
+  current_timezone_contexts.append(tzc)
+
   return "Timezone Updated"
+
+
+def resetTimeZone():
+  """Reset the timezone that might have been set by `setTimezone`
+  """
+  current_timezone_contexts.pop().__exit__(None, None, None)

product/ERP5/bootstrap/erp5_core/ModuleComponentTemplateItem/portal_components/module.erp5.DateUtils.py

@@ -27,11 +27,18 @@
 #
 ##############################################################################
 
+import contextlib
+import os
+from datetime import datetime
+from string import zfill
+import sys
+import time
 import warnings
 
+import mock
+import pytz
 from AccessControl import ModuleSecurityInfo
 from DateTime import DateTime
-from datetime import datetime
 import six
 
 security = ModuleSecurityInfo(__name__)
@@ -523,3 +530,35 @@ def copyDate(date, year=None, month=None, day=None,
   return DateTime('%i/%i/%i %i:%i:%d %s' % (year, month, day,
                                             hour, minute, second,
                                             timezone))
+
+
+@contextlib.contextmanager
+def timeZoneContext(timezone):
+  """Context manager to change timezone in tests.
+  """
+  saved_TZ = os.environ.get('TZ')
+  os.environ['TZ'] = timezone
+  time.tzset()
+  if timezone in pytz.all_timezones:
+    _multipleZones = time.daylight
+    _localzone0 = time.tzname[0]
+    _localzone1 = time.tzname[1] if time.daylight else time.tzname[0]
+  else:
+    _multipleZones = False
+    _localzone0 = _localzone1 = timezone
+  if hasattr(sys.modules['DateTime.DateTime'].DateTime, '_localzone0'):
+    patch_target = sys.modules['DateTime.DateTime'].DateTime
+  else:
+    # BBB DateTime 2
+    patch_target = sys.modules['DateTime.DateTime']
+
+  try:
+    with mock.patch.object(patch_target, '_localzone0', new=_localzone0), \
+        mock.patch.object(patch_target, '_localzone1', new=_localzone1), \
+        mock.patch.object(patch_target, '_multipleZones', new=_multipleZones):
+      yield
+  finally:
+    os.environ.pop('TZ')
+    if saved_TZ:
+      os.environ['TZ'] = saved_TZ
+    time.tzset()

product/ERP5/mixin/periodicity.py

@@ -234,7 +234,11 @@ class PeriodicityMixin:
     """
     returns something like ['Sunday','Monday',...]
     """
-    return DateTime._days
+    try:
+      from DateTime.DateTime import _DAYS
+      return _DAYS
+    except ImportError: # BBB DateTime 2.12
+      return DateTime._days
 
   security.declareProtected(Permissions.AccessContentsInformation, 'getWeekDayItemList')
   def getWeekDayItemList(self):

product/ERP5OOo/tests/testOOoImport.py

@@ -241,7 +241,7 @@ class TestOOoImport(TestOOoImportMixin):
       sorted(['male' for i in range(num)]),
       sorted([person_list[i].getGender() for i in range(num)]))
     self.assertEqual(
-      sorted([DateTime('2008/02/%02d %s' % (i+1, 'GMT')) for i in range(num)]),
+      sorted([DateTime('2008-02-%02d' % (i+1)) for i in range(num)]),
       sorted([person_list[i].getStartDate() for i in range(num)]))
 
   def stepCheckImportFloatsAndPercentage(self, sequence=None, sequence_list=None, **kw):

product/ERP5Type/patches/DateTimePatch.py

@@ -29,23 +29,29 @@ from __future__ import print_function
 
 from DateTime import DateTime as DateTimeKlass
 import math
-from DateTime.DateTime import _calcSD, _calcDependentSecond, _calcYMDHMS,\
-getDefaultDateFormat, _correctYear, _calcHMS, _calcDependentSecond2, DateTimeError,\
-SyntaxError, DateError, TimeError, localtime, time
+from DateTime.DateTime import _calcSD, _calcDependentSecond, _calcYMDHMS, \
+  DateTimeError, SyntaxError, DateError, TimeError
 
 STATE_KEY = 'str'
 
 original_DateTime__setstate__ = DateTimeKlass.__setstate__
 
 def DateTime__setstate__(self, state):
-  self.__dict__.clear()
-  if isinstance(state, tuple):
+  try: # BBB DateTime 2.12.8
+    self.__dict__.clear()
+  except AttributeError:
+    pass
+  if isinstance(state, tuple) and len(state) == 2:
+    _timezone_naive = False
     t, tz = state
+    if isinstance(tz, tuple):
+      tz, _timezone_naive = tz
     ms = (t - math.floor(t))
     s,d = _calcSD(t)
     x = _calcDependentSecond(tz, t)
     yr, mo, dy, hr, mn, sc = _calcYMDHMS(x, ms)
     self._parse_args(yr, mo, dy, hr, mn, sc, tz, t, d, s)
+    self._timezone_naive = _timezone_naive
   elif len(state) != 1 or STATE_KEY not in state:
     # For original pickle representation
     original_DateTime__setstate__(self, state)
@@ -56,211 +62,17 @@ def DateTime__setstate__(self, state):
 DateTimeKlass.__setstate__ = DateTime__setstate__
 
 def DateTime__getstate__(self):
+  if self.timezoneNaive():
+    return (self._t, (self._tz, self.timezoneNaive()))
   return (self._t, self._tz)
 
 DateTimeKlass.__getstate__ = DateTime__getstate__
 
-def DateTime_parse(self, st, datefmt=getDefaultDateFormat()):
-  # Parse date-time components from a string
-  month=year=tz=tm=None
-  spaces        =self.space_chars
-  intpat        =self.int_pattern
-  fltpat        =self.flt_pattern
-  wordpat       =self.name_pattern
-  delimiters    =self.delimiters
-  MonthNumbers  =self._monthmap
-  DayOfWeekNames=self._daymap
-  ValidZones    =self._tzinfo._zidx
-  TimeModifiers =['am','pm']
 
-  # Find timezone first, since it should always be the last
-  # element, and may contain a slash, confusing the parser.
-  st= st.strip()
-  sp=st.split()
-  tz=sp[-1]
-  if tz and (tz.lower() in ValidZones): st=' '.join(sp[:-1])
-  else: tz = None  # Decide later, since the default time zone
-  # could depend on the date.
-
-  ints,dels=[],[]
-  i,l=0,len(st)
-  while i < l:
-    while i < l and st[i] in spaces    : i=i+1
-    if i < l and st[i] in delimiters:
-      d=st[i]
-      i=i+1
-    else: d=''
-    while i < l and st[i] in spaces    : i=i+1
-
-    # The float pattern needs to look back 1 character, because it
-    # actually looks for a preceding colon like ':33.33'. This is
-    # needed to avoid accidentally matching the date part of a
-    # dot-separated date string such as '1999.12.31'.
-    if i > 0: b=i-1
-    else: b=i
-
-    ts_results = fltpat.match(st, b)
-    if ts_results:
-      s=ts_results.group(1)
-      i=i+len(s)
-      ints.append(float(s))
-      continue
-
-    #AJ
-    ts_results = intpat.match(st, i)
-    if ts_results:
-      s=ts_results.group(0)
-
-      ls=len(s)
-      i=i+ls
-      if (ls==4 and d and d in '+-' and
-          (len(ints) + bool(month) >= 3)):
-          tz='%s%s' % (d,s)
-      else:
-          v=int(s)
-          ints.append(v)
-      continue
-
-
-    ts_results = wordpat.match(st, i)
-    if ts_results:
-      o,s=ts_results.group(0),ts_results.group(0).lower()
-      i=i+len(s)
-      if i < l and st[i]=='.': i=i+1
-      # Check for month name:
-      if s in MonthNumbers:
-        v=MonthNumbers[s]
-        if month is None:
-          month = v
-          continue
-      # Check for time modifier:
-      elif s in TimeModifiers:
-        if tm is None:
-          tm = s
-          continue
-      # Check for and skip day of week:
-      elif s in DayOfWeekNames:
-        continue
-
-    raise SyntaxError(st)
-
-  day=None
-  if ints[-1] > 60 and d not in ['.',':','/'] and len(ints) > 2:
-    year=ints[-1]
-    del ints[-1]
-    if month:
-      day=ints[0]
-      del ints[:1]
-    else:
-      month=ints[0]
-      day=ints[1]
-      del ints[:2]
-  elif month:
-    if len(ints) > 1:
-      if ints[0] > 31:
-        year=ints[0]
-        day=ints[1]
-      else:
-        year=ints[1]
-        day=ints[0]
-      del ints[:2]
-  elif len(ints) > 2:
-    if ints[0] > 31:
-      year=ints[0]
-      if ints[1] > 12:
-        day=ints[1]
-        month=ints[2]
-      else:
-        day=ints[2]
-        month=ints[1]
-    if ints[1] > 31:
-      year=ints[1]
-      if ints[0] > 12 and ints[2] <= 12:
-        day=ints[0]
-        month=ints[2]
-      elif ints[2] > 12 and ints[0] <= 12:
-        day=ints[2]
-        month=ints[0]
-    elif ints[2] > 31:
-      year=ints[2]
-      if ints[0] > 12:
-        day=ints[0]
-        month=ints[1]
-      else:
-        if datefmt=="us":
-          day=ints[1]
-          month=ints[0]
-        else:
-          day=ints[0]
-          month=ints[1]
-
-    elif ints[0] <= 12:
-      month=ints[0]
-      day=ints[1]
-      year=ints[2]
-    del ints[:3]
-
-  if day is None:
-    # Use today's date.
-    year,month,day = localtime(time())[:3]
-
-  year = _correctYear(year)
-  #handle dates before year 1000
-  #if year < 1000: raise SyntaxError, st
-
-  leap = year%4==0 and (year%100!=0 or year%400==0)
-  try:
-    if not day or day > self._month_len[leap][month]:
-      raise DateError(st)
-  except IndexError:
-    raise DateError(st)
-  tod=0
-  if ints:
-    i=ints[0]
-    # Modify hour to reflect am/pm
-    if tm and (tm=='pm') and i<12:  i=i+12
-    if tm and (tm=='am') and i==12: i=0
-    if i > 24: raise TimeError(st)
-    tod = tod + int(i) * 3600
-    del ints[0]
-    if ints:
-      i=ints[0]
-      if i > 60: raise TimeError(st)
-      tod = tod + int(i) * 60
-      del ints[0]
-      if ints:
-        i=ints[0]
-        if i > 60: raise TimeError(st)
-        tod = tod + i
-        del ints[0]
-        if ints: raise SyntaxError(st)
-
-
-  tod_int = int(math.floor(tod))
-  ms = tod - tod_int
-  hr,mn,sc = _calcHMS(tod_int, ms)
-  if not tz:
-    # Figure out what time zone it is in the local area
-    # on the given date.
-    x = _calcDependentSecond2(year,month,day,hr,mn,sc)
-    tz = self._calcTimezoneName(x, ms)
-
-  return year,month,day,hr,mn,sc,tz
-
-DateTimeKlass._parse = DateTime_parse
-
-if __name__ == '__main__':
-  for i in ('2007/01/02 12:34:56.789',
-            '2007/01/02 12:34:56.789 GMT+0200',
-            '2007/01/02 12:34:56.789 JST',
-            '2007/01/02 12:34:56.789 +0300',
-            '2007/01/02 12:34:56.789 +0430',
-            '2007/01/02 12:34:56.789 +1237',
-            ):
-    a = DateTimeKlass(i)
-    b = DateTimeKlass()
-    b.__setstate__(a.__getstate__())
-    print(a, a.__dict__ == b.__dict__)
-    for i in a.__dict__.keys():
-      if a.__dict__[i] != b.__dict__[i]:
-        print(i, a.__dict__[i], b.__dict__[i])
+# DateTime 3 removed exceptions as class attributes (since
+# zopefoundation/DateTime commit 8114618 ), but we have some code expecting
+# these attributes, so undo this patch for convenience.
+DateTimeKlass.DateTimeError = DateTimeError
+DateTimeKlass.SyntaxError = SyntaxError
+DateTimeKlass.DateError = DateError
+DateTimeKlass.TimeError = TimeError

product/ERP5Type/tests/ERP5TypeTestCase.py

@@ -28,6 +28,7 @@ from glob import glob
 from hashlib import md5
 from warnings import warn
 from DateTime import DateTime
+import mock
 import Products.ZMySQLDA.DA
 from Products.ZMySQLDA.DA import Connection as ZMySQLDA_Connection
 
@@ -383,15 +384,11 @@ class ERP5TypeTestCaseMixin(ProcessingNodeTestCase, PortalTestCase, unittest.Tes
       self.pinDateTime(None)
 
     def setTimeZoneToUTC(self):
-      # Make sure tests runs with UTC timezone. Some tests are checking values
-      # based on now, and this could give unexpected results:
-      # DateTime("2016/10/31") - DateTime("2016/10/30") = 1.0416666666666667 if
-      # you are running on a timezone like Europe/Paris, while it return 1.0 for
-      # UTC
-      os.environ['TZ'] = "UTC"
-      time.tzset()
-      DateTime._isDST = False
-      DateTime._localzone = DateTime._localzone0 = DateTime._localzone1 = "UTC"
+      # Deprecated, prefer using `timeZoneContext` context manager instead.
+      from erp5.component.module.DateUtils import timeZoneContext
+      timezone = timeZoneContext('UTC')
+      timezone.__enter__()
+      self.addCleanup(timezone.__exit__, None, None, None)
 
     def getDefaultSystemPreference(self):
       id = 'default_system_preference'

product/ZSQLCatalog/SearchKey/DateTimeKey.py

@@ -31,11 +31,18 @@ from __future__ import absolute_import
 
 import six
 from six import string_types as basestring
+import calendar
 from .SearchKey import SearchKey
 from Products.ZSQLCatalog.Query.SimpleQuery import SimpleQuery
 from Products.ZSQLCatalog.Query.ComplexQuery import ComplexQuery
 from zLOG import LOG
-from DateTime.DateTime import DateTime, DateTimeError, _cache
+from DateTime.DateTime import DateTime, DateTimeError
+from DateTime import Timezones
+try:
+  from DateTime.DateTime import _TZINFO as _cache
+except ImportError:
+  # BBB DateTime 2
+  from DateTime.DateTime import _cache
 from Products.ZSQLCatalog.interfaces.search_key import ISearchKey
 from zope.interface.verify import verifyClass
 from Products.ZSQLCatalog.SearchText import parse
@@ -106,7 +113,7 @@ def castDate(value, change_timezone=True):
 delimiter_list = ' -/.:,+'
 
 def getMonthLen(datetime):
-  return datetime._month_len[datetime.isLeapYear()][datetime.month()]
+  return calendar.monthrange(datetime.year(), datetime.month())[1]
 
 def getYearLen(datetime):
   return 365 + datetime.isLeapYear()