Fix both ways sync

.gitignore

@@ -9,3 +9,7 @@ content
 hugo/resources
 hugo_stats.json
 
+.wgd-directory.yaml
+.tree.json
+.wgd-local-links.csv
+.wgd-local-log.csv

src/LinkTranslator.ts

@@ -2,7 +2,7 @@
 
 import RelateUrl from 'relateurl';
 
-import {LinkMode} from './model/model';
+import {LinkMode} from './model/model.ts';
 
 export function convertExtension(localPath: string, mode?: LinkMode) {
   const lastSlash = localPath.lastIndexOf('/');

src/google/markdownToHtml.ts

@@ -5,7 +5,7 @@ export async function markdownToHtml(buffer: Buffer): Promise<string> {
   const renderer = {
     paragraph(text: string) {
       return `<p>${text}</p><br />\n`;
-    }
+    },
     // code(code: string, infostring: string | undefined, escaped: boolean) {
     //   if (code.endsWith('\n')) {
     //     code = code + '\n';
@@ -18,12 +18,13 @@ export async function markdownToHtml(buffer: Buffer): Promise<string> {
     // image(href: string, title: string, text: string) {
     //   return `<img alt="${title}" src="${href}" />`;
     // },
-    // heading(text, level) {
-    //   const escapedText = text.toLowerCase().replace(/[^\w]+/g, '-');
-    //   return `
-    //         <h${level}><a id="${escapedText}" class="anchor" href="#${escapedText}"><span class="header-link"></span></a>${text}
-    //         </h${level}>`;
-    // }
+    heading(text, level) {
+      const escapedText = text.toLowerCase().replace(/[^\w]+/g, ' ').trim().replaceAll(' ', '-');
+      return `<h${level} id="${escapedText}">${text}</h${level}>\n`;
+      // return `
+      //        <h${level}><a id="${escapedText}" class="anchor" href="#${escapedText}"><span class="header-link"></span></a>${text}
+      //        </h${level}>`;
+    }
   };
 
   marked.use({ renderer });

src/odt/postprocess/convertGoogleUrls.ts

@@ -0,0 +1,11 @@
+import {MarkdownNodes} from '../MarkdownNodes.ts';
+import {walkRecursiveSync} from '../markdownNodesUtils.ts';
+import {replaceUrlsWithIds} from '../../utils/idParsers.ts';
+
+export function convertGoogleUrls(markdownChunks: MarkdownNodes) {
+  walkRecursiveSync(markdownChunks.body, (chunk) => {
+    if ('text' in chunk) {
+      chunk.text = replaceUrlsWithIds(chunk.text);
+    }
+  });
+}

src/odt/postprocess/fixIdLinks.ts

@@ -0,0 +1,31 @@
+import {MarkdownNodes} from '../MarkdownNodes.ts';
+import {extractText, walkRecursiveSync} from '../markdownNodesUtils.ts';
+
+export function fixIdLinks(markdownChunks: MarkdownNodes) {
+  let inHtml = false;
+  walkRecursiveSync(markdownChunks.body, (chunk) => {
+    if (chunk.isTag && chunk.tag === 'HTML_MODE/') {
+      inHtml = true;
+      return;
+    }
+
+    if (inHtml) {
+      return;
+    }
+
+    if (chunk.isTag && 'A' === chunk.tag) {
+      if (chunk.payload?.href && chunk.payload?.href.startsWith('#')) {
+        const innerTxt = extractText(chunk);
+        const escapedText = innerTxt.toLowerCase().replace(/[^\w]+/g, ' ').trim().replaceAll(' ', '-');
+        if (escapedText) {
+          chunk.payload.href = '#' + escapedText;
+        }
+      }
+    }
+  }, {}, (chunk) => {
+    if (chunk.isTag && chunk.tag === 'HTML_MODE/') {
+      inHtml = false;
+      return;
+    }
+  });
+}

src/odt/postprocess/postProcess.ts

@@ -25,6 +25,8 @@ import {applyRewriteRules} from './applyRewriteRules.ts';
 import {RewriteRule} from '../applyRewriteRule.ts';
 import {convertMathMl} from './convertMathMl.ts';
 import {unwrapEmptyPre} from './unwrapEmptyPre.ts';
+import {convertGoogleUrls} from './convertGoogleUrls.ts';
+import {fixIdLinks} from './fixIdLinks.ts';
 
 export async function postProcess(chunks: MarkdownNodes, rewriteRules: RewriteRule[]) {
   convertToc(chunks);
@@ -52,8 +54,11 @@ export async function postProcess(chunks: MarkdownNodes, rewriteRules: RewriteRu
   removeEmptyTags(chunks);
   addEmptyLines(chunks);
 
+  fixIdLinks(chunks);
   removeExcessiveLines(chunks);
 
+  convertGoogleUrls(chunks);
+
   applyRewriteRules(chunks, rewriteRules);
 
   if (process.env.DEBUG_COLORS) {

src/odt/postprocess/processListsAndNumbering.ts

@@ -155,7 +155,7 @@ export function processListsAndNumbering(markdownChunks: MarkdownNodes) {
     }
 
     return { ...ctx, level: ctx.level + 1 };
-  }, { level: 0 }, (chunk, ctx: { level: number }) => {
+  }, { level: 0 }, (chunk) => {
     if (!chunk.isTag) {
       return;
     }

src/utils/idParsers.ts

@@ -15,6 +15,12 @@ function stripUrlSuffix(id) {
   return null;
 }
 
+export function replaceUrlsWithIds(text: string): string {
+  text = text.replaceAll('https://drive.google.com/open?id%3D', 'https://drive.google.com/open?id=');
+  text = text.replaceAll('https://drive.google.com/open?id=', 'gdoc:');
+  return text;
+}
+
 export function urlToFolderId(url: string): string | null {
   if (!url) {
     return null;

test/md_html/header_links.html

@@ -0,0 +1,15 @@
+<html>
+<head>
+  <meta content="text/html; charset=UTF-8" http-equiv="content-type" />
+  <style>
+  .code { font-family: Courier, serif; }
+  </style>
+  </head>
+<body>
+<h1 id="developer-guide">Developer Guide</h1>
+<p>See <a href="#node-setup-on-the-system">Node setup on the system</a> for prereq.</p><br />
+<p><a href="https://drive.google.com/open?id=0AIkOKXbzWCtSUk9PVA">Example Google Drive Shared Folder</a></p><br />
+<h1 id="node-setup-on-the-system">Node setup on the system</h1>
+<h2 id="using-os">using OS</h2>
+</body>
+</html>

test/md_html/header_links.md

@@ -0,0 +1,9 @@
+# Developer Guide
+
+See [Node setup on the system](#node-setup-on-the-system) for prereq.
+
+[Example Google Drive Shared Folder](https://drive.google.com/open?id=0AIkOKXbzWCtSUk9PVA)
+
+# Node setup on the system
+
+## using OS

test/md_html/markdownToHtmlTest.ts

@@ -48,7 +48,7 @@ describe('markdownToHtml', () => {
     assert.equal(map['../subdir/diagram.svg'], '/aaa/subdir/diagram.svg');
 
     const serilzd = render(dom);
-    console.log(serilzd);
+    assert.ok(!!serilzd);
   });
 
   it('test markdownToHtml pre', async () => {
@@ -59,10 +59,8 @@ describe('markdownToHtml', () => {
 
     const dom = htmlparser2.parseDocument(html);
 
-    console.log(markdown);
-
     const serilzd = render(dom);
-    console.log(serilzd);
+    assert.ok(!!serilzd);
   });
 
   it('test markdownToHtml link_to_image', async () => {
@@ -73,8 +71,6 @@ describe('markdownToHtml', () => {
 
     const dom = htmlparser2.parseDocument(html);
 
-    console.log(markdown);
-
     const serilzd = render(dom);
     console.log(serilzd);
   });
@@ -85,4 +81,10 @@ describe('markdownToHtml', () => {
     assert.ok(compareTexts(testHtml, markdown));
   });
 
+  it('test ./header_links', async () => {
+    const testHtml = fs.readFileSync(__dirname + '/header_links.html').toString();
+    const markdown = await transformMd('header_links');
+    assert.ok(compareTexts(testHtml, markdown));
+  });
+
 });

test/md_html/paras.html

@@ -6,7 +6,7 @@
   </style>
 </head>
 <body>
-<h3>Why not make a website front end to a Google shared drive?</h3>
+<h3 id="why-not-make-a-website-front-end-to-a-google-shared-drive">Why not make a website front end to a Google shared drive?</h3>
 <p>Our goals are to be able to take versions of the content and commit them along with a version of the code at a point in time. By just making a website, it would allow for real-time viewing of the content but no way to go to a specific version of the documentation at a given time.</p><br />
 <p>A website front end is a goal for real-time testing of the viewing experience, but initially, we want to make markdown that can be committed.</p><br />
 </body>

test/odt_md/Issues.test.ts

@@ -66,6 +66,13 @@ describe('MarkDownTransformTest', () => {
     assert.ok(compareTexts(testMarkdown, markdown, false, 'our-docs.md'));
   });
 
+  it('test ./header-link', async () => {
+    // https://github.com/mieweb/wikiGDrive/issues/443
+    const testMarkdown = fs.readFileSync(__dirname + '/header-link.md').toString();
+    const markdown = await transformOdt('header-link');
+    assert.ok(compareTexts(testMarkdown, markdown, false, 'header-link.md'));
+  });
+
 });
 
 async function transformOdt(id: string) {

test/odt_md/header-link.md

@@ -0,0 +1,14 @@
+# Developer Guide
+
+See [Node setup on the system](#node-setup-on-the-system) for prereq.
+
+[Example Google Drive Shared Folder](gdoc:0AIkOKXbzWCtSUk9PVA)
+
+# Node setup on the system
+
+## using OS
+
+```
+curl -sL https://deb.nodesource.com/setup_16.x | sudo bash -
+sudo apt install nodejs
+```

test/odt_md/list-indent.md

@@ -3,7 +3,7 @@
 
 ![](10000201000004BE0000011D30E30AE192655040.png)
 
-3. When adding action items to panels, the [Representative Event panel action](#tyjcwt) is usually added to the panel first. Fill out all of the necessary fields according to the information acquired in the Health Surveillance matrix, and click <strong>Submit</strong> to save the panel action to the panel.
+3. When adding action items to panels, the [Representative Event panel action](#representative-event-panel-action) is usually added to the panel first. Fill out all of the necessary fields according to the information acquired in the Health Surveillance matrix, and click <strong>Submit</strong> to save the panel action to the panel.
     1. <strong>Action Name</strong>: Required field. The Action Name is usually the name of a test/procedure that is the component/action of the panel. The name will be displayed listings and dialogues throughout the system.
     2. <strong>Lead Time</strong>: The Lead Time translates to the number of days prior to the Trigger Date the panel action becomes visible and is created within the system. This defines how many days before the Trigger Date that the panel/orders will populate on the Due List. Keep Lead Times consistent when setting multiple action items in a panel; otherwise, each component of the panel will have different Due Dates if there are different Lead Times on each. Emails can be configured to send email notifications, as needed, with a list of associated charts/employees that will be due. The recipient has the time between receiving the email and the panel action Trigger Date to notify Health Services of any issues or mistakes with the list. Emails to the member/chart will not be sent until the actual Trigger Date. (Email reminders are separately configured on a per client basis. Email notification may not apply to all clients).
 
@@ -13,7 +13,7 @@ If the panel action is for a type of exposure, users will not want to set any Le
 {{% /tip %}}
 
     3. <strong>Required for Certification</strong>: Select this to indicate the panel action is required for members of the panel. Leave unchecked if the panel action is voluntary. If checked, a panel member failing or becoming overdue for the action will become de-certified from the panel.
-    4. <strong>Indication Rule</strong>: Users can select any action rule found in the Action Rules editor, using the drop-down. For more information on the Action Rules, see the [Health Surveillance Action Rules](gdoc:10wTqIF8gtUDBbJmbk_LjlUeNmtU_vvbVFoVWTZnuMqc) documentation. The action rule must evaluate to <strong>True</strong> in order for this panel action to trigger for a panel member. [Action Rules](#1fob9te) are usually configured by an MIE Developer after an MIE Implementer has collected all of the necessary details for the configuration.
+    4. <strong>Indication Rule</strong>: Users can select any action rule found in the Action Rules editor, using the drop-down. For more information on the Action Rules, see the [Health Surveillance Action Rules](gdoc:10wTqIF8gtUDBbJmbk_LjlUeNmtU_vvbVFoVWTZnuMqc) documentation. The action rule must evaluate to <strong>True</strong> in order for this panel action to trigger for a panel member. [Action Rules](#action-rules) are usually configured by an MIE Developer after an MIE Implementer has collected all of the necessary details for the configuration.
         1. Indication Rules can be used to only trigger the panel action for a member of the panel, if they are part of a specific department, for example. Or another more complex example would be a panel action configured to trigger a Hep3rd injection, only if the member of the panel had the second Hepatitis injection given within the last 8 weeks.
     5. <strong>Contraindication Rule</strong>: Users can select any action rule found in the Action Rules editor, using the drop-down. The action rule must evaluate to <strong>False</strong> in order for this panel action to trigger for a panel member. For more information on the Action Rules, see the [Health Surveillance Action Rules](gdoc:10wTqIF8gtUDBbJmbk_LjlUeNmtU_vvbVFoVWTZnuMqc) documentation.
     6. <strong>Trigger Type</strong>: Entry, Routine, Exit. Select the type of trigger, to define at what point in the panel member's current role/job status, the regulating agency or company requires the panel action to be completed. Entry will trigger when a panel member is first put in the panel. The Panel Evaluator scheduled job will run every day, triggering panels as appropriate, based on the the configured panel actions and the trigger type selected.

test/odt_md/our-docs.md

@@ -6,7 +6,7 @@ Google Drive to MarkDown synchronization
 
 WikiGDrive is a node app that uses the [Google Drive API](https://developers.google.com/drive/api/v3/quickstart/nodejs) to transform Google Docs and Drawings into markdown.
 
-![Diagram](https://drive.google.com/open?id=1TjB_v1gcBy23MtBQKvh5FZ3fFUp520QZ)
+![Diagram](gdoc:1TjB_v1gcBy23MtBQKvh5FZ3fFUp520QZ)
 
 [Google Drive Notes](gdoc:1H6vwfQXIexdg4ldfaoPUjhOZPnSkNn6h29WD6Fi-SBY) | [Github Project](https://github.com/mieweb/wikiGDrive/projects) | [Github Developer Notes](gdoc:1NJUxTnJHgkMO3JV1v_DFPsVsl8nDhweyDvNW0y98TGs)
 

test/odt_md/project-overview.md

@@ -6,9 +6,9 @@
   * [Collisions with Filenames](#collisions-with-filenames)
   * [Table of Contents and Index](#table-of-contents-and-index)
     * [Table of Contents](#table-of-contents)
-    * [Index](#_25nwvh7c83vs)
+    * [Index](#index)
   * [Markdown Cleanup](#markdown-cleanup)
-  * [Macros to Hugo Shortcodes (issue)](#_m5135xwpqj94)
+  * [Macros to Hugo Shortcodes (issue)](#macros-to-hugo-shortcodes-issue)
   * [Images](#images)
   * [FAQ](#faq)
 
@@ -22,9 +22,9 @@
   * [Collisions with Filenames](#collisions-with-filenames)
   * [Table of Contents and Index](#table-of-contents-and-index)
     * [Table of Contents](#table-of-contents) 
-    * [Index](#_25nwvh7c83vs)
+    * [Index](#index)
   * [Markdown Cleanup](#markdown-cleanup)
-  * [Macros to Hugo Shortcodes (issue)](#_m5135xwpqj94)
+  * [Macros to Hugo Shortcodes (issue)](#macros-to-hugo-shortcodes-issue)
   * [Images](#images)
   * [FAQ](#faq)
 
@@ -66,16 +66,16 @@ The app must:
 
 1. be able to be run once or run as a daemon (looking for changes in near real-time)
 2. Take changes from gdrive and propagate them down to the local file system (likely a git repo)
-3. Detect file [moves and renames](#renames-and-redirecting) from prior runs and leave redirects in the local file system to the new file or directory. 
+3. Detect file [moves and renames](#moves-and-renames) from prior runs and leave redirects in the local file system to the new file or directory. 
 4. Convert google docs to markdown while preserving as much of the meaning of the document. (Headings, images, drawings, tables, etc). 
     1. Each generated file should have parsable comments embedded in the original source google doc is known. 
     2. Embedded images (not originally stored on the shared folder will have to be extracted to the filesystem with a hashing system to prevent duplicate copies files in cases where images are pasted into multiple documents.
 5. Convert google drawings to svg and fix up urls as well. 
 6. Download images and place them in the proper folder. Embed metadata in the image pointing to the source on the google drive. It could be a .md file with the same name as the image. 
 7. Translate hyperlinks to the filesystem relative paths if they exist in the shared drive (both within Docs and Drawings). Must support both document urls and heading URLs. 
-8. Construct a [table of contents and an index](#table-of-contents-and-index) from all of the documents in the shared drive.
+8. Construct a [table of contents and an index](#table-of-contents-and-an-index) from all of the documents in the shared drive.
     1. It should be parsable so Javascript on the client could search and build navigation 
-    2. There should be generated markdown file ([toc.md](#table-of-contents) and [index.md](#_25nwvh7c83vs))
+    2. There should be generated markdown file ([toc.md](#toc-md) and [index.md](#index-md))
 
 Later phase:
 

test/odt_md/strong-headers.md

@@ -22,7 +22,7 @@ Some clients require the migration of historical, or non-active HS memberships.
 
 #### Open Orders
 
-In some cases, open or pending orders are required for migration for employees with overdue but active panel memberships. This use case is discussed further in the section on [How to determine the Next Due Date on a HS Panel](#3rdcrjn).
+In some cases, open or pending orders are required for migration for employees with overdue but active panel memberships. This use case is discussed further in the section on [How to determine the Next Due Date on a HS Panel](#how-to-determine-the-next-due-date-on-a-hs-panel).
 
 #### Historical Orders
 

website/docs/developer-guide.md

@@ -4,7 +4,7 @@ navWeight: -15
 ---
 # Developer Guide
 
-See [Node setup on the system](#Node-setup-on-the-system) for prereq.
+See [Node setup on the system](#node-setup-on-the-system) for prereq.
 
 [Example Google Drive Shared Folder](https://drive.google.com/open?id=0AIkOKXbzWCtSUk9PVA)