Skip to content

Commit

Permalink
Merge pull request #19 from contentstack/snippets
Browse files Browse the repository at this point in the history
Global fields support
  • Loading branch information
rohitmishra209 authored Nov 15, 2019
2 parents 62aff18 + ba7ffa6 commit d709451
Show file tree
Hide file tree
Showing 12 changed files with 214 additions and 42 deletions.
3 changes: 2 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ Update configuration details at config/index.js
## Usage
Once all things are configured, you can run following commands
1. Import all modules [ assets, locales, environments, extensions, webhooks, content_types, entries ]
1. Import all modules [ assets, locales, environments, extensions, webhooks, global_fields, content_types, entries ]
```bash
$ npm run import
```
Expand All @@ -41,6 +41,7 @@ $ npm run import-locales
$ npm run import-env
$ npm run import-extensions
$ npm run import-webhooks
$ npm run import-globalfields
$ npm run import-contenttypes
$ npm run import-entries
```
Expand Down
18 changes: 17 additions & 1 deletion config/default.js
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ module.exports = {
'environments',
'extensions',
'webhooks',
'global_fields',
'content_types',
'entries'
],
Expand Down Expand Up @@ -76,6 +77,20 @@ module.exports = {
limit: 50,
assetBatchLimit: 5
},

globalfields: {
dirName: 'global_fields',
fileName: 'globalfields.json',
validKeys: [
'title',
'uid',
'schema',
'options',
'singleton',
'description'
],
limit: 100
},
stack: {
dirName: 'stack',
fileName: 'stack.json'
Expand All @@ -90,13 +105,14 @@ module.exports = {
entries: '/entries/',
extensions: '/extensions/',
webhooks: '/webhooks/',
globalfields: '/global_fields/',
folders: '/folders/',
stacks: '/stacks/'
},
preserveStackVersion: false
// if exisstingContentDir exists, no backup folder will be created
// rather, its value(path) will be used instead
// useBackedupDir: './_backup_007',
// useBackedupDir: './_backup_340',
// is the no. of files to be copied/backed up concurrently
// backupConcurrency: 10,
};
3 changes: 2 additions & 1 deletion config/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,10 @@ module.exports = {
name: 'English - United States',
code: 'en-us'
},
// Credentials
email: '',
password: '',
target_stack: '',
// Folder in which exported contents are stored
data: '../contentstack-export/_contents',
data: './_contents_1_All'
};
54 changes: 28 additions & 26 deletions lib/import/content_types.js
Original file line number Diff line number Diff line change
Expand Up @@ -19,14 +19,14 @@ var config = app.getConfig();
var contentTypeConfig = config.modules.content_types;
var contentTypesFolderPath = path.resolve(config.data, contentTypeConfig.dirName);
//var contentTypesFolderPath = path.resolve("/home/rohit/Test-import-export/contentstack-import/_backup_381", "/content_types");
var extensionPath = path.resolve(config.data, "mapper/extensions", "uid-mapping.json");
//var extensionPath = path.resolve(config.data, "mapper/extensions", "uid-mapping.json");
var mapperFolderPath = path.join(config.data, 'mapper', 'content_types');
var skipFiles = ['__master.json', '__priority.json', 'schema.json'];
var fileNames = fs.readdirSync(path.join(contentTypesFolderPath));
var field_rules_ct = [];


function importContentTypes () {
function importContentTypes() {
var self = this;

this.contentTypes = [];
Expand All @@ -48,7 +48,7 @@ function importContentTypes () {
}
this.contentTypeUids = _.difference(this.contentTypeUids, this.createdContentTypeUids);
// remove contet types, already created
_.remove(this.contentTypes, function (contentType) {
_.remove(this.contentTypes, function(contentType) {
return self.contentTypeUids.indexOf(contentType.uid) === -1;
});
this.schemaTemplate = require('../util/schemaTemplate');
Expand All @@ -61,48 +61,47 @@ function importContentTypes () {
}

importContentTypes.prototype = {
start: function () {
start: function() {
var self = this;

return new Promise(function (resolve, reject) {
return Promise.map(self.contentTypeUids, function (contentTypeUid) {
return self.seedContentTypes(contentTypeUid).then(function () {
return new Promise(function(resolve, reject) {
return Promise.map(self.contentTypeUids, function(contentTypeUid) {
return self.seedContentTypes(contentTypeUid).then(function() {
return;
}).catch(reject);
}, {
// seed 3 content types at a time
concurrency: 3
}).then(function () {
}).then(function() {
// content type seeidng completed
self.requestOptions.method = 'PUT';
return Promise.map(self.contentTypes, function (contentType) {
return self.updateContentTypes(contentType).then(function () {
return Promise.map(self.contentTypes, function(contentType) {
return self.updateContentTypes(contentType).then(function() {
log.success(contentType.uid + ' was updated successfully!');
return;
}).catch(reject);
}).then(function () {
fs.writeFile(contentTypesFolderPath + "/field_rules_uid.json", JSON.stringify(field_rules_ct),
function (err) {
if (err) throw err;
});
}).then(function() {
fs.writeFile(contentTypesFolderPath + '/field_rules_uid.json', JSON.stringify(field_rules_ct), function(err) {
if (err) throw err;
});
log.success('Content types have been imported successfully!');
// content types have been successfully imported
return resolve();
}).catch(reject);
}).catch(reject);
});
},
seedContentTypes: function (uid) {
seedContentTypes: function(uid) {
var self = this;
return new Promise(function (resolve, reject) {
return new Promise(function(resolve, reject) {
var body = _.cloneDeep(self.schemaTemplate);
body.content_type.uid = uid;
body.content_type.title = uid;
var requestObject = _.cloneDeep(self.requestOptions);
requestObject.json = body;
return request(requestObject)
.then(resolve)
.catch(function (error) {
.catch(function(error) {
if (error.error_code === 115 && (error.errors.uid || error.errors.title)) {
// content type uid already exists
return resolve();
Expand All @@ -111,23 +110,26 @@ importContentTypes.prototype = {
});
});
},
updateContentTypes: function (contentType) {
updateContentTypes: function(contentType) {
var self = this;
return new Promise(function (resolve, reject) {
return new Promise(function(resolve, reject) {
var requestObject = _.cloneDeep(self.requestOptions);
requestObject.uri += contentType.uid;
contentTypeschema = contentType.schema
//contentTypeschema = contentType.schema;
if (contentType.field_rules) {
field_rules_ct.push(contentType.uid)
field_rules_ct.push(contentType.uid);
delete contentType.field_rules;
}
supress(contentType.schema)
supress(contentType.schema);
requestObject.json.content_type = contentType;
return request(requestObject).then(function (response) {
return request(requestObject).then(function(response) {
self.createdContentTypeUids.push(response.body.content_type.uid);
helper.writeFile(path.join(mapperFolderPath, 'success.json'), self.createdContentTypeUids);
return resolve();
}).catch(function (error) {})
}).catch(function(error) {
log.error(error);
return reject(error);
});
});
}
};
Expand Down
4 changes: 2 additions & 2 deletions lib/import/environments.js
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,8 @@ var app = require('../../app');
var config = app.getConfig();
var environmentConfig = config.modules.environments;
var environmentsFolderPath = path.resolve(config.data, environmentConfig.dirName);
var envMapperPath = path.resolve(config.data, 'environments');
var envUidMapperPath = path.resolve(config.data, 'environments', 'uid-mapping.json');
var envMapperPath = path.resolve(config.data, 'mapper', 'environments');
var envUidMapperPath = path.resolve(config.data, 'mapper', 'environments', 'uid-mapping.json');
var envSuccessPath = path.resolve(config.data, 'environments', 'success.json');
var envFailsPath = path.resolve(config.data, 'environments', 'fails.json');

Expand Down
6 changes: 3 additions & 3 deletions lib/import/extensions.js
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,8 @@ var app = require('../../app');
var config = app.getConfig();
var extensionsConfig = config.modules.extensions;
var extensionsFolderPath = path.resolve(config.data, extensionsConfig.dirName);
var extMapperPath = path.resolve(config.data, 'mapper', "extensions");
var extUidMapperPath = path.resolve(config.data, "mapper/extensions", 'uid-mapping.json');
var extMapperPath = path.resolve(config.data, 'mapper', 'extensions');
var extUidMapperPath = path.resolve(config.data, 'mapper/extensions', 'uid-mapping.json');
var extSuccessPath = path.resolve(config.data, 'extensions', 'success.json');
var extFailsPath = path.resolve(config.data, 'extensions', 'fails.json');

Expand Down Expand Up @@ -61,7 +61,7 @@ importExtensions.prototype = {
headers: self.requestOptions.headers,
method: self.requestOptions.method,
json: {
extension: ext
extension: ext
}
};

Expand Down
109 changes: 109 additions & 0 deletions lib/import/global_fields.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
/*!
* Contentstack Import
* Copyright (c) 2019 Contentstack LLC
* MIT Licensed
*/

var mkdirp = require('mkdirp');
var fs = require('fs');
var path = require('path');
var Promise = require('bluebird');

var request = require('../util/request');
var helper = require('../util/fs');
var log = require('../util/log');
var app = require('../../app');
var extension_supress = require('../util/extensionsUidReplace');

var config = app.getConfig();
var globalfieldsConfig = config.modules.globalfields;
var globalfieldsFolderPath = path.resolve(config.data, globalfieldsConfig.dirName);
var globalfieldsMapperPath = path.resolve(config.data, 'mapper', 'global_fields');
var globalfieldsUidMapperPath = path.resolve(config.data, 'mapper', 'global_fields', 'uid-mapping.json');
var globalfieldsSuccessPath = path.resolve(config.data, 'mapper', 'global_fields', 'success.json');
var globalfieldsFailsPath = path.resolve(config.data, 'mapper', 'global_fields', 'fails.json');

if (!fs.existsSync(globalfieldsMapperPath)) {
mkdirp.sync(globalfieldsMapperPath);
}

function importGlobalFields () {
this.fails = [];
this.success = [];
this.snipUidMapper = {};
this.globalfields = helper.readFile(path.resolve(globalfieldsFolderPath, globalfieldsConfig.fileName));
if (fs.existsSync(globalfieldsUidMapperPath)) {
this.snipUidMapper = helper.readFile(globalfieldsUidMapperPath);
this.snipUidMapper = this.snipUidMapper || {};
}
this.requestOptions = {
uri: config.host + config.apis.globalfields,
headers: config.headers,
method: 'POST'
};
}

importGlobalFields.prototype = {
start: function () {
var self = this;
return new Promise(function (resolve, reject) {
if(self.globalfields == undefined) {
log.success('No globalfields Found');
return resolve();
}
var snipUids = Object.keys(self.globalfields);
return Promise.map(snipUids, function (snipUid) {
var snip = self.globalfields[snipUid];
extension_supress(snip.schema);

if (!self.snipUidMapper.hasOwnProperty(snipUid)) {
var requestOption = {
uri: self.requestOptions.uri,
headers: self.requestOptions.headers,
method: self.requestOptions.method,
json: {
global_field: snip
}
};

// return self.createglobalfieldssnipUidMapper
return request(requestOption).then(function (response) {
self.success.push(response.body.global_field);
var global_field_uid = response.body.global_field.uid;
self.snipUidMapper[snipUid] = global_field_uid;
helper.writeFile(globalfieldsUidMapperPath, self.snipUidMapper);
log.success(global_field_uid +' '+' globalfield created successfully');
return;
}).catch(function (error) {
if(error.error_code === 115) {
// eslint-disable-next-line no-console
console.log(error.message);
}
self.fails.push(snip);
// log.error('globalfields failed to be imported\n' + error);
return;
});
} else {
// the globalfields has already been created
log.success('The globalfields already exists. Skipping it to avoid duplicates!');
return;
}
// import 2 globalfields at a time
}, {
concurrency: 2
}).then(function () {
// globalfields have imported successfully
helper.writeFile(globalfieldsSuccessPath, self.success);
// log.success('globalfields have been imported successfully!');
return resolve();
}).catch(function (error) {
// error while importing globalfields
helper.writeFile(globalfieldsFailsPath, self.fails);
log.error('globalfields import failed');
return reject(error);
});
});
}
};

module.exports = new importGlobalFields();
30 changes: 30 additions & 0 deletions lib/import/locales.js
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ importLanguages.prototype = {
}
};
return request(requestOption).then(function (response) {
self.update_locales(lang);
self.success.push(response.body.locale);
self.langUidMapper[langUid] = response.body.locale.uid;
helper.writeFile(langUidMapperPath, self.langUidMapper);
Expand Down Expand Up @@ -98,6 +99,35 @@ importLanguages.prototype = {
log.error('Language import failed');
return reject(error);
});

});
},
update_locales: function(lang) {
var self = this;
var requestOption = {
uri: self.requestOptions.uri+lang.code,
headers: self.requestOptions.headers,
method: 'PUT',
json: {
locale: {
code: lang.code,
fallback_locale: lang.fallback_locale,
name: lang.name
}
}
};

return request(requestOption).then(function () {
return;

}).catch(function (error) {
if (error.hasOwnProperty('error_code') && error.error_code === 247) {
log.success(error.errors.code[0]);
return;
}
self.fails.push(lang);
log.error('Language: \'' + lang.code + '\' failed to be imported\n' + error);
throw error;
});
}
};
Expand Down
Loading

0 comments on commit d709451

Please sign in to comment.