from Products.ERP5Type.tests.SecurityTestCase import SecurityTestCase
import string
import random
import time
from DateTime import DateTime
import numpy as np
import math
import base64
from Products.ERP5Type.Log import log

class TestDataIngestion(SecurityTestCase):

  PART = "/xxx"
  EOF = "/EOF"
  FIF = "/fif"
  REF_PREFIX = "fake-supplier/fake-dataset/"
  INGESTION_SCRIPT = 'HandleFifEmbulkIngestion'
  USER = 'zope'
  PASS = 'roque5'

  def getTitle(self):
    return "DataIngestionTest"

  def getBusinessTemplateList(self):
    return ('erp5_base', 'erp5_web', 'erp5_ingestion_mysql_innodb_catalog', 'erp5_ingestion', 'erp5_dms',
            'erp5_wendelin', 'erp5_callables', 'erp5_core')

  def afterSetUp(self):
    self.context = self.portal.UnitTest_getContext()

  def getRandomReference(self):
    random_string = ''.join([random.choice(string.ascii_letters + string.digits) for _ in xrange(10)])
    return 'UNIT-TEST-' + random_string

  def getIngestionReference(self, reference):
    return self.REF_PREFIX + reference + self.FIF

  def sanitizeReference(self, reference):
    return reference.replace("/", "_"), '/'.join(reference.split('/')[1:])

  def chunks(self, l, n):
    for i in xrange(0, len(l), n):
      yield l[i:i+n]

  def generateRawDataBytesAndArray(self):
    array = np.random.randn(5, 1000)
    raw_data = self.context.generateRawData(array)
    return raw_data, array

  def getIngestionPolicy(self, reference, ingestion_script):
    ingestion_policy = self.portal.portal_ingestion_policies.newContent( \
        id = reference,
        portal_type ='Ingestion Policy',
        reference = reference,
        version = '001',
        script_id = ingestion_script)
    ingestion_policy.validate()
    self.tic()
    return ingestion_policy

  def ingestRequest(self, requestMethod, authentication , reference, eof, data_chunk, ingestion):
    encoded_data_chunk = base64.b64encode(data_chunk)
    request = self.portal.REQUEST
    request.method = requestMethod
    request.auth = authentication
    request.set('reference', reference + eof)
    request.set('data_chunk', encoded_data_chunk)
    ingestion.ingest()
    self.tic()
    return

  def getDataStream(self, reference):
    data_stream = self.portal.portal_catalog.getResultValue(
                    portal_type = 'Data Stream',
                    reference = reference)
    return data_stream

  def getDataArray(self, reference):
    data_array = self.portal.portal_catalog.getResultValue(
                    portal_type = 'Data Array',
                    reference = reference)
    return data_array

  def manuallyStopIngestionWorkaround(self, reference, now):
    data_ingestion_id =  "%s_%s" %(reference, now)
    url = 'data_ingestion_module/' + data_ingestion_id
    data_ingestion = self.context.restrictedTraverse(url)
    if data_ingestion.getSimulationState() == "started":
      data_ingestion.stop()
    self.tic()

  def simulateIngestionAlarm(self, reference, now):
    self.portal.ERP5Site_stopIngestionList()
    self.tic()
    self.manuallyStopIngestionWorkaround(reference, now)

    self.portal.ERP5Site_createDataAnalysisList()
    time.sleep(5)
    self.tic()

    self.portal.ERP5Site_executeDataAnalysisList()
    time.sleep(5)
    self.tic()

  def test_full_data_ingestion(self):
    reference = self.getRandomReference()
    ingestion_policy = self.getIngestionPolicy(reference, self.INGESTION_SCRIPT)
    data_chunk, nparray = self.generateRawDataBytesAndArray()
    ingestion_reference = self.getIngestionReference(reference)

    now = DateTime().strftime('%Y%m%d-%H%M%S')
    self.ingestRequest('POST', (self.USER, self.PASS), ingestion_reference, self.EOF, data_chunk, ingestion_policy)
    ingestion_id, ingestion_reference = self.sanitizeReference(ingestion_reference)
    data_stream = self.getDataStream(ingestion_reference)
    self.assertEqual(len(data_chunk), len(data_stream.getData()))
    self.assertEqual(data_chunk, data_stream.getData())

    self.simulateIngestionAlarm(ingestion_id, now)

    data_array = self.getDataArray(ingestion_reference)
    self.assertTrue(np.allclose(nparray, data_array.getArray()))


  def test_data_ingestion_splitted_file(self):
    reference = self.getRandomReference()
    ingestion_policy = self.getIngestionPolicy(reference, self.INGESTION_SCRIPT)
    data_chunk, nparray = self.generateRawDataBytesAndArray()
    data_chunk_1 = data_chunk[:int(math.floor(len(data_chunk)/2))]
    data_chunk_2 = data_chunk[int(math.floor(len(data_chunk)/2)):]
    ingestion_reference = self.getIngestionReference(reference)

    self.ingestRequest('POST', (self.USER, self.PASS), ingestion_reference, self.PART, data_chunk_1, ingestion_policy)
    self.ingestRequest('POST', (self.USER, self.PASS), ingestion_reference, self.EOF, data_chunk_2, ingestion_policy)
    ingestion_id, ingestion_reference = self.sanitizeReference(ingestion_reference)
    data_stream = self.getDataStream(ingestion_reference)
    self.assertEqual(len(data_chunk), len(data_stream.getData()))
    self.assertEqual(data_chunk, data_stream.getData())

























