Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

wip: add node links and backreferences during node creation #1

Closed
wants to merge 24 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
b5a615d
use node references where applicable
nocash Dec 14, 2018
5521efe
allow async arrow functions for eslint
nocash Dec 15, 2018
2aafc47
fetch from Ghost /tags and /users endpoints
nocash Dec 15, 2018
c7f0671
create backreferences during node creation
nocash Dec 17, 2018
9e29735
rewrite api functions to share more code
nocash Dec 17, 2018
a74abad
use sinon object as sandbox
nocash Dec 15, 2018
c5c9f43
update main test so that it passes
nocash Dec 17, 2018
96b78d8
remove GhostPage type
nocash Dec 17, 2018
e15ae5a
remove empty line at eof
nocash Dec 17, 2018
2e9117f
convert image urls to media nodes w/ local files
nocash Dec 18, 2018
b8b990e
Merge branch 'create-node-references' into develop
nocash Dec 19, 2018
48bb89b
add gatsby-source-filesystem as peer dependency
nocash Dec 19, 2018
b9fbe82
Merge branch 'create-node-references' into develop
nocash Dec 19, 2018
9d94070
set gatsby-source-filesystem as dependency
nocash Dec 19, 2018
34cf236
Merge branch 'create-node-references' into develop
nocash Dec 19, 2018
7519e5b
Add og_image and twitter_image as GhostMedia nodes
nsimonson Dec 21, 2018
576448e
Merge remote-tracking branch 'origin/create-node-references' into dev…
nsimonson Jan 1, 2019
5ce8374
Merge tag 'v2.2.0' from upstream
nocash Jan 11, 2019
5c825a4
Merge tag 'v3.0.0' from upstream
nocash Jan 12, 2019
7ac51d4
move image processing up in code to make it harder to miss
nocash Jan 14, 2019
a9826f3
Merge tag 'v3.1.0' from upstreak
nocash Jan 14, 2019
03787b7
Merge tag 'v3.1.1' from upstream
nocash Jan 14, 2019
adee06e
Merge remote-tracking branch 'TryGhost/master'
nocash Jan 14, 2019
55f0f5d
Merge branch 'develop' into create-node-references
nocash Jan 14, 2019
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .eslintrc.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,7 @@
module.exports = {
parserOptions: {
ecmaVersion: 8,
},
plugins: ['ghost'],
extends: [
'plugin:ghost/node',
Expand Down
70 changes: 43 additions & 27 deletions gatsby-node.js
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
const GhostContentAPI = require('@tryghost/content-api');
const Promise = require('bluebird');
const {PostNode, PageNode, TagNode, AuthorNode, SettingsNode} = require('./nodes');
const {createNodeFactories} = require('./nodes');
const {getImagesFromApiResults} = require('./lib');

exports.sourceNodes = async ({actions, createNodeId, store, cache}, configOptions) => {
const {createNode, touchNode} = actions;
const imageArgs = {createNode, createNodeId, touchNode, store, cache};

exports.sourceNodes = ({boundActionCreators}, configOptions) => {
const {createNode} = boundActionCreators;
const api = new GhostContentAPI({
host: configOptions.apiUrl,
key: configOptions.contentApiKey,
Expand All @@ -16,34 +18,48 @@ exports.sourceNodes = ({boundActionCreators}, configOptions) => {
formats: 'html,plaintext'
};

const fetchPosts = api.posts.browse(postAndPageFetchOptions).then((posts) => {
posts.forEach(post => createNode(PostNode(post)));
});

const fetchPages = api.pages.browse(postAndPageFetchOptions).then((pages) => {
pages.forEach(page => createNode(PageNode(page)));
});

const tagAndAuthorFetchOptions = {
limit: 'all',
include: 'count.posts'
include: 'count.posts,count.pages'
};

const fetchTags = api.tags.browse(tagAndAuthorFetchOptions).then((tags) => {
tags.forEach((tag) => {
tag.postCount = tag.count.posts;
createNode(TagNode(tag));
});
});
const [posts, pages, tags, users, settings] = await Promise.all([
api.posts.browse(postAndPageFetchOptions),
api.pages.browse(postAndPageFetchOptions),
api.tags.browse(tagAndAuthorFetchOptions),
api.authors.browse(tagAndAuthorFetchOptions),
api.settings.browse()
]);

const fetchAuthors = api.authors.browse(tagAndAuthorFetchOptions).then((authors) => {
authors.forEach((author) => {
author.postCount = author.count.posts;
createNode(AuthorNode(author));
});
});
const {
buildPostNode,
buildPageNode,
buildTagNode,
buildAuthorNode,
buildSettingsNode,
buildMediaNode
} = createNodeFactories({posts, pages, tags, users}, imageArgs);

const images = getImagesFromApiResults([posts, pages, tags, users]);
for (const image of images) {
createNode(await buildMediaNode(image));
}

for (const post of posts) {
createNode(await buildPostNode(post));
}

for (const page of pages) {
createNode(await buildPageNode(page));
}

for (const tag of tags) {
createNode(buildTagNode(tag));
}

const fetchSettings = api.settings.browse().then(setting => createNode(SettingsNode(setting)));
for (const user of users) {
createNode(buildAuthorNode(user));
}

return Promise.all([fetchPosts, fetchPages, fetchTags, fetchAuthors, fetchSettings]);
createNode(buildSettingsNode(settings));
};
14 changes: 14 additions & 0 deletions lib.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
module.exports.getImagesFromApiResults = results => results
.reduce((acc, entities) => acc.concat(entities))
.reduce(
(acc, entity) => acc.concat([
entity.feature_image,
entity.cover_image,
entity.profile_image,
entity.og_image,
entity.twitter_image
]),
[]
)
.filter(url => !!url)
.map(url => ({id: url, src: url}));
256 changes: 247 additions & 9 deletions nodes.js
Original file line number Diff line number Diff line change
@@ -1,19 +1,257 @@
const createNodeHelpers = require('gatsby-node-helpers').default;
const {createRemoteFileNode} = require('gatsby-source-filesystem');

const {
createNodeFactory
} = createNodeHelpers({
typePrefix: 'Ghost'
const TYPE_PREFIX = 'Ghost';
const {createNodeFactory, generateNodeId} = createNodeHelpers({
typePrefix: TYPE_PREFIX
});

const POST = 'Post';
const PAGE = 'Page';
const TAG = 'Tag';
const AUTHOR = 'Author';
const SETTINGS = 'Settings';
const MEDIA = 'Media';

module.exports.PostNode = createNodeFactory(POST);
module.exports.PageNode = createNodeFactory(PAGE);
module.exports.TagNode = createNodeFactory(TAG);
module.exports.AuthorNode = createNodeFactory(AUTHOR);
module.exports.SettingsNode = createNodeFactory(SETTINGS);
async function downloadImageAndCreateFileNode(
{url},
{createNode, createNodeId, touchNode, store, cache}
) {
let fileNodeID;

const mediaDataCacheKey = `${TYPE_PREFIX}__Media__${url}`;
const cacheMediaData = await cache.get(mediaDataCacheKey);

if (cacheMediaData) {
fileNodeID = cacheMediaData.fileNodeID;
touchNode({nodeId: fileNodeID});
return fileNodeID;
}

const fileNode = await createRemoteFileNode({
url,
store,
cache,
createNode,
createNodeId
});

if (fileNode) {
fileNodeID = fileNode.id;
await cache.set(mediaDataCacheKey, {fileNodeID});
return fileNodeID;
}
}

function mapPostToTags(post, tags) {
const postHasTags =
post.tags && Array.isArray(post.tags) && post.tags.length;

if (postHasTags) {
// replace tags with links to their nodes
post.tags___NODE = post.tags.map(t => generateNodeId(TAG, t.id));

// add a backreference for this post to the tags
post.tags.forEach(({id: tagId}) => {
const tag = tags.find(t => t.id === tagId);
if (!tag.posts___NODE) {
tag.posts___NODE = [];
}
tag.posts___NODE.push(post.id);
});

// replace primary_tag with a link to the tag node
if (post.primary_tag) {
post.primary_tag___NODE = generateNodeId(TAG, post.primary_tag.id);
}

delete post.tags;
delete post.primary_tag;
}
}

function mapPageToTags(page, tags) {
const pageHasTags =
page.tags && Array.isArray(page.tags) && page.tags.length;

if (pageHasTags) {
// replace tags with links to their nodes
page.tags___NODE = page.tags.map(t => generateNodeId(TAG, t.id));

// add a backreference for this post to the tags
page.tags.forEach(({id: tagId}) => {
const tag = tags.find(t => t.id === tagId);
if (!tag.pages___NODE) {
tag.pages___NODE = [];
}
tag.pages___NODE.push(page.id);
});

// replace primary_tag with a link to the tag node
if (page.primary_tag) {
page.primary_tag___NODE = generateNodeId(TAG, page.primary_tag.id);
}

delete page.tags;
delete page.primary_tag;
}
}

function mapPostToUsers(post, users) {
const postHasAuthors =
post.authors && Array.isArray(post.authors) && post.authors.length;

if (postHasAuthors) {
// replace authors with links to their (user) nodes
post.authors___NODE = post.authors.map(a => generateNodeId(AUTHOR, a.id)
);

// add a backreference for this post to the user
post.authors.forEach(({id: authorId}) => {
const user = users.find(u => u.id === authorId);
if (!user.posts___NODE) {
user.posts___NODE = [];
}
user.posts___NODE.push(post.id);
});

// replace primary_author with a link to the user node
if (post.primary_author) {
post.primary_author___NODE = generateNodeId(
AUTHOR,
post.primary_author.id
);
}

delete post.authors;
delete post.primary_author;
}
}

function mapPageToUsers(page, users) {
const pageHasAuthors =
page.authors && Array.isArray(page.authors) && page.authors.length;

if (pageHasAuthors) {
// replace authors with links to their (user) nodes
page.authors___NODE = page.authors.map(a => generateNodeId(AUTHOR, a.id)
);

// add a backreference for this post to the user
page.authors.forEach(({id: authorId}) => {
const user = users.find(u => u.id === authorId);
if (!user.pages___NODE) {
user.pages___NODE = [];
}
user.pages___NODE.push(page.id);
});

// replace primary_author with a link to the user node
if (page.primary_author) {
page.primary_author___NODE = generateNodeId(
AUTHOR,
page.primary_author.id
);
}

delete page.authors;
delete page.primary_author;
}
}

async function mapImagesToMedia(node) {
if (node.feature_image) {
node.feature_image___NODE = generateNodeId(MEDIA, node.feature_image);
delete node.feature_image;
}

if (node.profile_image) {
node.profile_image___NODE = generateNodeId(MEDIA, node.profile_image);
delete node.profile_image;
}

if (node.cover_image) {
node.cover_image___NODE = generateNodeId(MEDIA, node.cover_image);
delete node.cover_image;
}

if (node.og_image) {
node.og_image___NODE = generateNodeId(MEDIA, node.og_image);
delete node.og_image;
}

if (node.twitter_image) {
node.twitter_image___NODE = generateNodeId(MEDIA, node.twitter_image);
delete node.twitter_image;
}
}

function addPostCountToTag(tag, posts) {
tag.postCount = posts.reduce((acc, post) => {
const postHasTag = post.tags && !!post.tags.find(pt => tag.ghostId === pt.id);
return postHasTag ? acc + 1 : acc;
}, 0);
}

function addPostCountToAuthor(author, posts) {
author.postCount = posts.reduce((acc, post) => {
const postHasAuthor = post.authors && !!post.authors.find(pa => author.ghostId === pa.id);
return postHasAuthor ? acc + 1 : acc;
}, 0);
}

async function createLocalFileFromMedia(node, imageArgs) {
node.localFile___NODE = await downloadImageAndCreateFileNode(
{url: node.src.split('?')[0]},
imageArgs
);
}

module.exports.createNodeFactories = ({posts, tags, users}, imageArgs) => {
const postNodeMiddleware = (node) => {
mapPostToTags(node, tags);
mapPostToUsers(node, users);
mapImagesToMedia(node);
return node;
};

const pageNodeMiddleware = (node) => {
mapPageToTags(node, tags);
mapPageToUsers(node, users);
mapImagesToMedia(node);
return node;
};

const tagNodeMiddleware = (node) => {
addPostCountToTag(node, posts);
mapImagesToMedia(node);
return node;
};

const authorNodeMiddleware = (node) => {
addPostCountToAuthor(node, posts);
mapImagesToMedia(node);
return node;
};

const mediaNodeMiddleware = async (node) => {
await createLocalFileFromMedia(node, imageArgs);
return node;
};

const buildPostNode = createNodeFactory(POST, postNodeMiddleware);
const buildPageNode = createNodeFactory(PAGE, pageNodeMiddleware);
const buildTagNode = createNodeFactory(TAG, tagNodeMiddleware);
const buildAuthorNode = createNodeFactory(AUTHOR, authorNodeMiddleware);
const buildSettingsNode = createNodeFactory(SETTINGS);
const buildMediaNode = createNodeFactory(MEDIA, mediaNodeMiddleware);

return {
buildPostNode,
buildPageNode,
buildTagNode,
buildAuthorNode,
buildSettingsNode,
buildMediaNode
};
};
1 change: 1 addition & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
"@tryghost/content-api": "1.0.0",
"bluebird": "^3.5.1",
"gatsby-node-helpers": "^0.3.0",
"gatsby-source-filesystem": "^2.0.0",
"qs": "^6.5.2"
}
}
Loading