chore: WIP simplify data loading

astro.config.ts

@@ -7,11 +7,12 @@ import oembedTransformer from "@remark-embedder/transformer-oembed";
 import * as TwitchTransformer from "gatsby-remark-embedder/dist/transformers/Twitch.js";
 import rehypeSlug from "rehype-slug-custom-id";
 import { parent } from "./src/constants/site-config";
-import { rehypeHeaderText } from "./src/utils/markdown/plugins/rehype-header-text";
+import { rehypeHeaderText } from "./src/utils/markdown/rehype-header-text";
-import { rehypeTabs } from "./src/utils/markdown/plugins/tabs";
+import { rehypeTabs } from "./src/utils/markdown/tabs";
-import { rehypeAstroImageMd } from "./src/utils/markdown/plugins/rehype-astro-image-md";
+import { rehypeAstroImageMd } from "./src/utils/markdown/rehype-astro-image-md";
-import { rehypeUnicornElementMap } from "./src/utils/markdown/plugins/rehype-unicorn-element-map";
+import { rehypeUnicornElementMap } from "./src/utils/markdown/rehype-unicorn-element-map";
-import { rehypeExcerpt } from "./src/utils/markdown/plugins/rehype-excerpt";
+import { rehypeExcerpt } from "./src/utils/markdown/rehype-excerpt";
+import { rehypeUnicornPopulatePost } from "./src/utils/markdown/rehype-unicorn-populate-post";
 
 // TODO: Create types
 import behead from "remark-behead";
@@ -45,6 +46,7 @@ export default defineConfig({
       ],
     ],
     rehypePlugins: [
+      rehypeUnicornPopulatePost,
       // This is required to handle unsafe HTML embedded into Markdown
       rehypeRaw,
       // Do not add the tabs before the slug. We rely on some of the heading

src/components/post-card-list/post-card-list.astro

@@ -2,16 +2,18 @@
 import listStyle from "./post-card-list.module.scss";
 import PostCard from "../post-card/post-card.astro";
 // import { PostListContext } from "constants/post-list-context";
-import { ListViewPosts } from "utils/fs/api";
+// import { ListViewPosts } from "utils/fs/api";
 
 export interface PostListProps {
   showWordCount?: boolean;
   numberOfArticles?: number;
   wordCount?: number;
-  unicornData?: ListViewPosts[number]["authors"];
+  // unicornData?: ListViewPosts[number]["authors"];
+  unicornData?: any
   listAriaLabel: string;
   // Added
-  postsToDisplay: ListViewPosts
+  // postsToDisplay: ListViewPosts
+  postsToDisplay: any
 }
 /**
  * unicornData - The data with the associated post. If present - you're on profile page

src/components/post-card/post-card.astro

@@ -3,12 +3,13 @@
 // TODO: Make user-profile-pic clickable again
 import cardStyles from "./post-card.module.scss";
 import UserProfilePic from "../user-profile-pic/user-profile-pic.astro";
-import { ListViewPosts } from "utils/fs/api";
+// import { ListViewPosts } from "utils/fs/api";
 import dayjs from "dayjs";
 
 interface PostCardProps {
   title: string; // The title of the post
-  authors: ListViewPosts[number]["authors"]; // Info on the authors of the post
+  // authors: ListViewPosts[number]["authors"]; // Info on the authors of the post
+  authors: any; // Info on the authors of the post
   published: string; // Date the author published the post
   tags: string[]; // List of tags associated with the post
   excerpt: string; // The autogenerated excerpt

src/constants/queries.ts

@@ -1,32 +0,0 @@
-import { PickDeep } from "ts-util-helpers";
-import { PostInfo } from "types/PostInfo";
-
-export const seriesPostsPick = {
-  title: true,
-  slug: true,
-  series: true,
-  order: true,
-} as const;
-
-export type SeriesPostInfo = PickDeep<PostInfo, typeof seriesPostsPick>;
-
-export const postBySlug = {
-  title: true,
-  slug: true,
-  content: true,
-  wordCount: true,
-  series: true,
-  order: true,
-  originalLink: true,
-  tags: true,
-  edited: true,
-  published: true,
-  authors: true,
-  description: true,
-  excerpt: true,
-  license: true,
-  translations: true,
-  collectionSlug: true,
-} as const;
-
-export type SlugPostInfo = PickDeep<PostInfo, typeof postBySlug>;

src/page-components/post-list/post-list.astro

@@ -7,13 +7,14 @@ import PostList from "components/post-card-list/post-card-list.astro";
 // import { Pagination } from "components/pagination";
 // import { FilterSearchBar } from "components/filter-search-bar";
 import { siteMetadata } from "constants/site-config";
-import { ListViewPosts } from "utils/fs/api";
+// import { ListViewPosts } from "utils/fs/api";
 
 interface PostListTemplateProps {
   numberOfPages: number;
   limitNumber: number;
   pageIndex: number;
-  posts: ListViewPosts;
+  // posts: ListViewPosts;
+  posts: any;
 }
 
 const { 

src/types/CollectionInfo.ts

@@ -1,6 +1,5 @@
 import { UnicornInfo } from "./UnicornInfo";
 import { PostInfo } from "types/PostInfo";
-import { PickDeep } from "ts-util-helpers";
 
 export interface CollectionInfo {
   slug: string;

src/types/PostInfo.ts

@@ -3,28 +3,30 @@ import { LicenseInfo } from "./LicenseInfo";
 import { Languages } from "types/index";
 import { MarkdownInstance } from "astro";
 
-export interface PostInfo {
+export interface RawPostInfo {
+  title: string;
+  published: string;
+  authors: string[];
+  tags: string[];
+  attached: string[];
+  license: string;
+  description?: string;
+  edited?: string;
+  series?: string;
+  order?: number;
+  originalLink?: string;
+}
+
+export interface PostInfo extends RawPostInfo {
   slug: string;
   locale: Languages;
   Content: MarkdownInstance<never>['Content'];
-  title: string;
+  authorsMeta: UnicornInfo[];
-  published: string;
+  licenseMeta: LicenseInfo;
-  edited?: string;
-  authors: UnicornInfo[];
-  license: LicenseInfo;
   excerpt: string;
   wordCount: number;
-  description?: string;
-  series?: string;
   collectionSlug?: string | null;
-  order?: number;
-  originalLink?: string;
-  content: string;
-  tags: string[];
   translations: Partial<Record<Languages, string>>;
-}
-
-export interface RenderedPostInfo {
   headingsWithId?: Array<{
     // Title value
     value: string;
@@ -32,4 +34,4 @@ export interface RenderedPostInfo {
     slug: string;
     depth: number;
   }>;
-}
+}

src/types/UnicornInfo.ts

@@ -1,14 +1,12 @@
 import { PronounInfo } from "./PronounInfo";
 import { RolesEnum } from "./RolesInfo";
 
-export interface UnicornInfo {
+export interface RawUnicornInfo {
+  id: string;
   name: string;
   firstName: string;
   lastName: string;
-  id: string;
   description: string;
-  color: string;
-  roles: RolesEnum[];
   socials: {
     twitter?: string;
     github?: string;
@@ -17,8 +15,16 @@ export interface UnicornInfo {
     twitch?: string;
     dribbble?: string;
   };
-  pronouns: PronounInfo;
+  pronouns: string;
-  profileImg: {
+  profileImg: string;
+  color: string;
+  roles: Array<RolesEnum['id']>;
+}
+
+export interface UnicornInfo extends RawUnicornInfo {
+  rolesMeta: RolesEnum[];
+  pronounsMeta: PronounInfo;
+  profileImgMeta: {
     // Relative to "public/unicorns"
     relativePath: string;
     // Relative to site root

src/utils/api.ts

@@ -0,0 +1,43 @@
+import {
+  unicorns,
+  licenses,
+} from "../data";
+import { PostInfo } from "types/PostInfo";
+import { Languages } from "types/index";
+import { MarkdownInstance } from "astro";
+
+let allPostsCache = new WeakMap<object, PostInfo[]>();
+
+export function getAllPosts(
+  posts: MarkdownInstance<PostInfo>[],
+  language: Languages,
+  cacheString: null | object = null
+) {
+  if (cacheString) {
+    const cacheData = allPostsCache.get(cacheString);
+    if (cacheData) return cacheData as any;
+  }
+
+  if (cacheString) allPostsCache.set(cacheString, posts);
+
+  return posts
+  .filter(post => post.frontmatter.locale === language);
+}
+
+const listViewCache = {};
+
+export const getAllPostsForListView = (
+  posts: MarkdownInstance<PostInfo>[],
+  language: Languages,
+): PostInfo[] => {
+  let allPosts = getAllPosts(posts, language, listViewCache);
+
+  // sort posts by date in descending order
+  allPosts = allPosts.sort((post1, post2) => {
+    const date1 = new Date(post1.published);
+    const date2 = new Date(post2.published);
+    return date1 > date2 ? -1 : 1;
+  });
+
+  return allPosts;
+};

src/utils/data.ts

@@ -13,7 +13,7 @@ export const sponsorsDirectory = join(process.cwd(), "public/sponsors");
 const unicornsRaw: Array<
   Omit<UnicornInfo, "roles" | "pronouns" | "profileImg"> & {
     roles: string[];
-    pronouns?: string;
+    pronouns: string;
     profileImg: string;
   }
 > = JSON.parse(
@@ -43,7 +43,7 @@ const fullUnicorns: UnicornInfo[] = unicornsRaw.map((unicorn) => {
   // Mutation go BRR
   const newUnicorn: UnicornInfo = unicorn as never;
 
-  newUnicorn.profileImg = {
+  newUnicorn.profileImgMeta = {
     height: profileImgSize.height as number,
     width: profileImgSize.width as number,
     relativePath: unicorn.profileImg,
@@ -51,20 +51,13 @@ const fullUnicorns: UnicornInfo[] = unicornsRaw.map((unicorn) => {
     absoluteFSPath,
   };
 
-  newUnicorn.roles = unicorn.roles.map(
+  newUnicorn.rolesMeta = unicorn.roles.map(
     (role) => rolesRaw.find((rRole) => rRole.id === role)!
   );
 
-  newUnicorn.pronouns = pronounsRaw.find(
+  newUnicorn.pronounsMeta = pronounsRaw.find(
-    (proWithNouns) => proWithNouns.id === unicorn?.pronouns
+    (proWithNouns) => proWithNouns.id === unicorn.pronouns
-  ) || {
+  )!;
-    id: "",
-    they: "",
-    them: "",
-    their: "",
-    theirs: "",
-    themselves: "",
-  };
 
   return newUnicorn;
 });

src/utils/fs/api.ts

@@ -1,21 +0,0 @@
-import {
-  dataDirectory,
-  unicorns,
-  pronouns,
-  licenses,
-  roles,
-  postsDirectory,
-  collectionsDirectory,
-} from "./get-datas";
-
-export * from "./posts-and-collections-api";
-
-export {
-  unicorns,
-  pronouns,
-  licenses,
-  roles,
-  dataDirectory,
-  postsDirectory,
-  collectionsDirectory,
-};

src/utils/fs/posts-and-collections-api.ts

@@ -1,120 +0,0 @@
-import {
-  unicorns,
-  licenses,
-} from "utils/fs/get-datas";
-import { PostInfo } from "types/PostInfo";
-import { getExcerpt } from "utils/markdown/getExcerpt";
-import { Languages } from "types/index";
-import { MarkdownInstance } from "astro";
-
-// const getIndexPath = (lang: Languages) => {
-//   const indexPath = lang !== "en" ? `index.${lang}.md` : `index.md`;
-//   return indexPath;
-// };
-
-export function extendPostMetadata(
-  post: MarkdownInstance<PostInfo>
-) {
-  // Split based on `/`, even in Windows
-  const directorySplit = post.file.split('/');
-
-  // This is the folder name, AKA how we generate the slug ID
-  const slug = directorySplit.at(-2);
- 
-  /** Calculate post locale */
-  // index.md or index.es.md
-  const indexName = directorySplit.at(-1);
-  const indexSplit = indexName.split('.');
-  let locale = indexSplit.at(-2);
-  if (locale === 'index') {
-    locale = 'en';
-  }
-
-  // // TODO: Add translations
-  // if (fields.translations) {
-  //   const langsToQuery: Languages[] = Object.keys(languages).filter(
-  //     (l) => l !== lang
-  //   ) as never;
-  //   pickedData.translations = langsToQuery
-  //     .filter((lang) =>
-  //       fs.existsSync(resolve(dirname(fullPath), getIndexPath(lang)))
-  //     )
-  //     .reduce((prev, lang) => {
-  //       prev[lang] = languages[lang];
-  //       return prev;
-  //     }, {} as Record<Languages, string>);
-  // }
-
-  // // TODO: Add collection slug
-  // if (fields.collectionSlug) {
-  //   if (frontmatterData.series) {
-  //     pickedData.collectionSlug = collectionsByName.find(
-  //       (collection) => collection.associatedSeries === frontmatterData.series
-  //     )?.slug;
-  //   }
-  //   if (!pickedData.collectionSlug) pickedData.collectionSlug = null;
-  // }
-
-  const authors = (post.frontmatter.authors as never as string[]).map(
-    (author) => unicorns.find((unicorn) => unicorn.id === author)!
-  );
-
-  let license;
-  if (post.frontmatter.license) {
-    license = licenses.find(
-      (l) => l.id === post.frontmatter.license as never as string
-    );
-  }
-  if (!license) license = null;
-
-  return {
-    ...post.frontmatter,
-    Content: post.Content,
-    slug,
-    locale,
-    authors,
-    license
-  } as PostInfo;
-}
-
-let allPostsCache = new WeakMap<object, PostInfo[]>();
-
-export function getAllPosts(
-  posts: MarkdownInstance<PostInfo>[],
-  language: Languages,
-  cacheString: null | object = null
-) {
-  if (cacheString) {
-    const cacheData = allPostsCache.get(cacheString);
-    if (cacheData) return cacheData as any;
-  }
-
-  // TODO: Move `Astro.glob` here
-  // const posts = await Astro.glob<PostInfo>('../../content/blog/**/*.md')
-
-  const newPosts = posts
-  .map(post => extendPostMetadata(post));
-  
-  if (cacheString) allPostsCache.set(cacheString, newPosts);
-
-  return newPosts
-  .filter(post => post.locale === language);
-}
-
-const listViewCache = {};
-
-export const getAllPostsForListView = (
-  posts: MarkdownInstance<PostInfo>[],
-  language: Languages,
-): PostInfo[] => {
-  let allPosts = getAllPosts(posts, language, listViewCache);
-
-  // sort posts by date in descending order
-  allPosts = allPosts.sort((post1, post2) => {
-    const date1 = new Date(post1.published);
-    const date2 = new Date(post2.published);
-    return date1 > date2 ? -1 : 1;
-  });
-
-  return allPosts;
-};

src/utils/markdown/count-words.ts

@@ -1,69 +0,0 @@
-import { unified } from "unified";
-import parse from "remark-parse";
-import stringify from "remark-stringify";
-import english from "retext-english";
-import remark2retext from "remark-retext";
-import {visit} from "unist-util-visit";
-import { Root, Node, Parent, Text } from "hast";
-import flatFilter from "unist-util-flat-filter";
-
-function count(counts: Record<string, number>) {
-  return () => counter;
-
-  function counter(tree: Root) {
-    visit(tree, visitor);
-
-    function visitor(node: Node) {
-      counts[node.type] = (counts[node.type] || 0) + 1;
-    }
-  }
-}
-
-function countInline(counts: Record<string, number>) {
-  return () => counter;
-
-  function counter(tree: Root) {
-    const inlineCodeAST = flatFilter(
-      tree,
-      (node) => node.type === "inlineCode"
-    ) as Parent;
-
-    counts["InlineCodeWords"] = 0;
-    if (inlineCodeAST && inlineCodeAST.children) {
-      counts["InlineCodeWords"] = inlineCodeAST.children.reduce(
-        (numberOfInline, inlineCodeNode) => {
-          const { value } = inlineCodeNode as Text;
-          const words = value.split(/\b/g);
-          return numberOfInline + words.length;
-        },
-        0
-      );
-    }
-
-    return tree;
-  }
-}
-
-export function countContent(content: string) {
-  const counts: Record<string, number> = {};
-
-  unified()
-    .use(parse)
-    .use(countInline(counts))
-    .use(remark2retext, unified().use(english).use(count(counts)))
-    .use(stringify)
-    .processSync(content);
-
-  return counts as {
-    InlineCodeWords: number;
-    RootNode: number;
-    ParagraphNode: number;
-    SentenceNode: number;
-    WordNode: number;
-    TextNode: number;
-    WhiteSpaceNode: number;
-    PunctuationNode: number;
-    SymbolNode: number;
-    SourceNode: number;
-  };;
-}

src/utils/markdown/rehype-astro-image-md.ts

@@ -10,8 +10,8 @@ import path from "path";
 /**
  * They need to be the same `getImage` with the same `globalThis` instance, thanks to the "hack" workaround.
  */
-import { getImage } from "../../../../node_modules/@astrojs/image";
+import { getImage } from "../../../node_modules/@astrojs/image";
-import sharp_service from "../../../../node_modules/@astrojs/image/dist/loaders/sharp.js";
+import sharp_service from "../../../node_modules/@astrojs/image/dist/loaders/sharp.js";
 
 interface RehypeAstroImageProps {
   maxHeight?: number;

src/utils/markdown/rehype-unicorn-element-map.ts

@@ -3,8 +3,8 @@ import { Plugin } from "unified";
 
 import { visit } from "unist-util-visit";
 
-import { EMBED_SIZE } from "../constants";
+import { EMBED_SIZE } from "./constants";
-import { isRelativePath } from "../../../utils/url-paths";
+import { isRelativePath } from "../../utils/url-paths";
 import { fromHtml } from "hast-util-from-html";
 
 import path from "path";

src/utils/markdown/rehype-unicorn-populate-post.ts

@@ -0,0 +1,81 @@
+import { Root } from "hast";
+import { Plugin } from "unified";
+import {visit} from 'unist-util-visit'
+import matter from "gray-matter";
+import { readFileSync } from "fs";
+import * as path from "path";
+import { licenses, unicorns } from "utils/data";
+
+interface RehypeUnicornPopulatePostProps {
+}
+
+export const rehypeUnicornPopulatePost: Plugin<
+  [RehypeUnicornPopulatePostProps | never],
+  Root
+> = () => {
+  return (tree, file) => {
+
+    function setData(key: string, val: any) {
+        (file.data.astro as any).frontmatter[key] = val;
+    }
+
+    const fileContents = readFileSync(file.path, "utf8");
+    const { data: frontmatter } = matter(fileContents);
+
+    const directorySplit = file.path.split(path.sep);
+
+    // This is the folder name, AKA how we generate the slug ID
+    const slug = directorySplit.at(-2);
+
+    // Calculate post locale
+    // index.md or index.es.md
+    const indexName = directorySplit.at(-1);
+    const indexSplit = indexName.split('.');
+    let locale = indexSplit.at(-2);
+    if (locale === 'index') {
+        locale = 'en';
+    }
+        
+    // // TODO: Add translations
+    // if (fields.translations) {
+    //   const langsToQuery: Languages[] = Object.keys(languages).filter(
+    //     (l) => l !== lang
+    //   ) as never;
+    //   pickedData.translations = langsToQuery
+    //     .filter((lang) =>
+    //       fs.existsSync(resolve(dirname(fullPath), getIndexPath(lang)))
+    //     )
+    //     .reduce((prev, lang) => {
+    //       prev[lang] = languages[lang];
+    //       return prev;
+    //     }, {} as Record<Languages, string>);
+    // }
+
+    // // TODO: Add collection slug
+    // if (fields.collectionSlug) {
+    //   if (frontmatterData.series) {
+    //     pickedData.collectionSlug = collectionsByName.find(
+    //       (collection) => collection.associatedSeries === frontmatterData.series
+    //     )?.slug;
+    //   }
+    //   if (!pickedData.collectionSlug) pickedData.collectionSlug = null;
+    // }
+
+    const authorsMeta = (frontmatter.authors as string[]).map(
+        (author) => unicorns.find((unicorn) => unicorn.id === author)!
+      );
+    
+      let license;
+      if (frontmatter.license) {
+        license = licenses.find(
+          (l) => l.id === frontmatter.license
+        );
+      }
+      if (!license) license = null;
+      
+      setData('slug', slug);
+      setData('locale', locale);
+      setData('authorsMeta', authorsMeta);
+      setData('license', license);
+  };
+};