From 11b35bab28f85633a9246c30fb7d908aa9f63a1b Mon Sep 17 00:00:00 2001 From: Tyrel Souza Date: Wed, 25 May 2016 12:46:03 -0400 Subject: [PATCH 1/4] Put flows in a subdirectory of each integration --- celigo/core.py | 63 +++++++++++++++++++++++++++++++++++--------------- tests.py | 8 ++++--- 2 files changed, 50 insertions(+), 21 deletions(-) diff --git a/celigo/core.py b/celigo/core.py index ba99de1..6006f55 100644 --- a/celigo/core.py +++ b/celigo/core.py @@ -1,4 +1,5 @@ import glob +import collections import logging import requests import io @@ -9,6 +10,7 @@ from slugify import slugify from .prompt import prompt +logging.basicConfig(level=logging.INFO) L = logging.getLogger(__name__) DEFAULT_BASE_URL = "https://api.integrator.io/v1/" @@ -76,9 +78,10 @@ class BackupCeligo(object): self.session = requests.Session() self.session.headers.update(self.headers) - def ensure_directories_exist(self): + def ensure_directories_exist(self, subdirs=None): """ Make the directory if it doesn't exist """ - subdirs = ('imports', 'connections') + if not subdirs: + subdirs = ('integrations', 'connections') for subdir in subdirs: _dir = os.path.join(self.data_dir, subdir) @@ -123,6 +126,17 @@ class BackupCeligo(object): varable. Once this is cached, save the imports. """ + try: + integrations = self._celigo_api_get("integrations/") + except requests.exceptions.RequestException: + L.info('HTTP Request failed') + raise + # Setup integrations dictionaries + for integration in integrations: + self.imports_cache[integration['_id']] = { + 'name': integration['name'], + 'slug': slugify(integration['name']), + 'flows': []} try: flows = self._celigo_api_get("flows/") for flow in flows: @@ -131,10 +145,16 @@ class BackupCeligo(object): L.info('HTTP Request failed') raise L.info("Got all imports, writing now") - L.info("We have imports for: %s", ", ".join(self.imports_cache.keys())) - for flow_name, (import_id, import_conf) in self.imports_cache.items(): - self.save_import(flow_name, auto) + for integration_id, integration in self.imports_cache.items(): + # Make sure the integration directories exist + base = "integrations/"+integration_id + subdirs = (base + "imports", base + "exports") + self.ensure_directories_exist(subdirs=subdirs) + + # save the files + for flow in integration['flows']: + self.save_flow(integration_id, flow, auto) def restore(self, auto=False): """ @@ -174,28 +194,35 @@ class BackupCeligo(object): """ Stores the import in self.imports_cache before write. """ - flow_name = slugify(flow['name']) + flow_name = flow['name'] import_id = flow['_importId'] + integration_id = flow['_integrationId'] import_conf = self._celigo_api_get( - "imports/{id}/distributed".format( - id=import_id)) + "imports/{id}/distributed".format(id=import_id)) - self.imports_cache[flow_name] = (import_id, import_conf) + self.imports_cache[integration_id]['flows'].append({ + "name": flow_name, + "id": import_id, + "configuration": import_conf + }) - def save_import(self, flow_name, auto=False): + def save_flow(self, integration_id, flow, auto=False): """ Write the import to a .json file with name_id.json format. Prompt for overwrite. - :param flow_name: the slugified name of the flow as a key + :param flow: dictionary of "name", "id", "configuration" for the + flow. :param auto: if auto is true, don't prompt for overwrite """ - import_id, import_conf = self.imports_cache[flow_name] - - filename = os.path.join( - self.data_dir, - "imports", - "%s_%s.json" % (flow_name, import_id)) + flow_name = flow['name'] + flow_id = flow['id'] + flow_conf = flow['configuration'] + filename = os.path.join(self.data_dir, + "integrations", + integration_id, + "imports", + "%s_%s.json" % (slugify(flow_name), flow_id)) write = True # By default, we prompt for overwrites @@ -209,7 +236,7 @@ class BackupCeligo(object): write = bool(overwrite == "Yes") if write: - self.write_json(filename, import_conf) + self.write_json(filename, flow_conf) else: L.info("You chose not to save this file.") diff --git a/tests.py b/tests.py index 540d611..99e3939 100644 --- a/tests.py +++ b/tests.py @@ -48,9 +48,10 @@ class CeligoTest(unittest.TestCase): """ Test that the ensure_directories_exist works properly. """ + fake_id = "fake_integration_id" tempdir = tempfile.mkdtemp('celigo_testing') bc = celigo.BackupCeligo(tempdir) - imports_dir = os.path.join(tempdir, "imports") + imports_dir = os.path.join(tempdir, fake_id, "imports") connections_dir = os.path.join(tempdir, "connections") # Check that the directories don't exist already. self.assertFalse( @@ -59,8 +60,9 @@ class CeligoTest(unittest.TestCase): self.assertFalse( os.path.exists(connections_dir), "connections dir exists") + # Make the directories. - bc.ensure_directories_exist() + bc.ensure_directories_exist((imports_dir, connections_dir)) self.assertTrue( os.path.exists(imports_dir), "Did not create proper directory" @@ -71,7 +73,7 @@ class CeligoTest(unittest.TestCase): ) # Make sure nothing errors if the directories exist already. - bc.ensure_directories_exist() + bc.ensure_directories_exist((imports_dir, connections_dir)) # @requests_mock.Mocker() # def test_fake_requests(self, rqm): From ec720e7ff415af350f73771a4241ca4438a039a0 Mon Sep 17 00:00:00 2001 From: Tyrel Souza Date: Wed, 25 May 2016 12:46:34 -0400 Subject: [PATCH 2/4] subdirect correctly --- tests.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests.py b/tests.py index 99e3939..6dc9a2c 100644 --- a/tests.py +++ b/tests.py @@ -51,7 +51,8 @@ class CeligoTest(unittest.TestCase): fake_id = "fake_integration_id" tempdir = tempfile.mkdtemp('celigo_testing') bc = celigo.BackupCeligo(tempdir) - imports_dir = os.path.join(tempdir, fake_id, "imports") + + imports_dir = os.path.join(tempdir, "integrations", fake_id, "imports") connections_dir = os.path.join(tempdir, "connections") # Check that the directories don't exist already. self.assertFalse( From ac188f42f567614fa491daff15a8a975509d1251 Mon Sep 17 00:00:00 2001 From: Tyrel Souza Date: Wed, 25 May 2016 13:07:23 -0400 Subject: [PATCH 3/4] actually make the real directory, not just ones in tests --- celigo/core.py | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/celigo/core.py b/celigo/core.py index 6006f55..e4c55e5 100644 --- a/celigo/core.py +++ b/celigo/core.py @@ -80,10 +80,8 @@ class BackupCeligo(object): def ensure_directories_exist(self, subdirs=None): """ Make the directory if it doesn't exist """ - if not subdirs: - subdirs = ('integrations', 'connections') - for subdir in subdirs: + L.info("Creating subdir: `%s`", subdir) _dir = os.path.join(self.data_dir, subdir) if not os.path.exists(_dir): os.makedirs(_dir) @@ -147,12 +145,6 @@ class BackupCeligo(object): L.info("Got all imports, writing now") for integration_id, integration in self.imports_cache.items(): - # Make sure the integration directories exist - base = "integrations/"+integration_id - subdirs = (base + "imports", base + "exports") - self.ensure_directories_exist(subdirs=subdirs) - - # save the files for flow in integration['flows']: self.save_flow(integration_id, flow, auto) @@ -190,6 +182,7 @@ class BackupCeligo(object): self.restore_to_celigo(action) del self.imports_cache[action] + def cache_import_remote(self, flow): """ Stores the import in self.imports_cache before write. @@ -198,7 +191,8 @@ class BackupCeligo(object): import_id = flow['_importId'] integration_id = flow['_integrationId'] import_conf = self._celigo_api_get( - "imports/{id}/distributed".format(id=import_id)) + "imports/{id}/distributed".format( + id=import_id)) self.imports_cache[integration_id]['flows'].append({ "name": flow_name, @@ -218,11 +212,16 @@ class BackupCeligo(object): flow_id = flow['id'] flow_conf = flow['configuration'] - filename = os.path.join(self.data_dir, - "integrations", - integration_id, - "imports", - "%s_%s.json" % (slugify(flow_name), flow_id)) + filedir = os.path.join(self.data_dir, + "integrations", + integration_id, + "imports") + + self.ensure_directories_exist((filedir,)) + + filename = os.path.join( + filedir, + "%s_%s.json" % (slugify(flow_name), flow_id)) write = True # By default, we prompt for overwrites From a5f7566361792b7e609ec9cdcf7a5372a2e74133 Mon Sep 17 00:00:00 2001 From: Tyrel Souza Date: Wed, 25 May 2016 13:28:10 -0400 Subject: [PATCH 4/4] Refactor, and update changelog --- Changelog | 6 ++++++ celigo/core.py | 24 ++++++++++++++++-------- tests.py | 4 +--- 3 files changed, 23 insertions(+), 11 deletions(-) create mode 100644 Changelog diff --git a/Changelog b/Changelog new file mode 100644 index 0000000..2a49cda --- /dev/null +++ b/Changelog @@ -0,0 +1,6 @@ +# Change Log + +## [Unreleased] +### Added +- Ability to download all imports/flows from Celigo +- Storing each flow in a subdirectory of an integration diff --git a/celigo/core.py b/celigo/core.py index e4c55e5..9309994 100644 --- a/celigo/core.py +++ b/celigo/core.py @@ -117,13 +117,7 @@ class BackupCeligo(object): L.info("Restored backup to %s", self.base_url + path) return response - def backup(self, auto=False): - """ - Get all the flow data from Celigo. - Then loop over each flow and cache it's Import data in an instance - varable. - Once this is cached, save the imports. - """ + def _get_integration_placeholders(self): try: integrations = self._celigo_api_get("integrations/") except requests.exceptions.RequestException: @@ -135,6 +129,8 @@ class BackupCeligo(object): 'name': integration['name'], 'slug': slugify(integration['name']), 'flows': []} + + def _get_flow_configurations(self): try: flows = self._celigo_api_get("flows/") for flow in flows: @@ -144,10 +140,23 @@ class BackupCeligo(object): raise L.info("Got all imports, writing now") + def _save_each_flow(self, auto=False): for integration_id, integration in self.imports_cache.items(): for flow in integration['flows']: self.save_flow(integration_id, flow, auto) + def backup(self, auto=False): + """ + Get all the flow data from Celigo. + Then loop over each flow and cache it's Import data in an instance + varable. + Once this is cached, save the imports. + """ + self._get_integration_placeholders() + self._get_flow_configurations() + self._save_each_flow(auto) + + def restore(self, auto=False): """ Get all the import files in the import direcotry, @@ -182,7 +191,6 @@ class BackupCeligo(object): self.restore_to_celigo(action) del self.imports_cache[action] - def cache_import_remote(self, flow): """ Stores the import in self.imports_cache before write. diff --git a/tests.py b/tests.py index 6dc9a2c..3eafdff 100644 --- a/tests.py +++ b/tests.py @@ -30,9 +30,7 @@ class CeligoTest(unittest.TestCase): """ data_dir = "fakedir" os.environ['CELIGO_API_KEY'] = '' - with self.assertRaisesRegexp( - Exception, - 'Please pass in api_key.*'): + with self.assertRaisesRegexp(Exception, 'Please pass in api_key.*'): celigo.BackupCeligo(data_dir) os.environ['CELIGO_API_KEY'] = self.FAKE_API_KEY bc = celigo.BackupCeligo(data_dir)