{"flow":{"id":50,"summary":"Sync Github Discussions to Discord Forum","versions":[199],"created_by":"rubenfiszel","created_at":"2023-08-18T14:00:13.166Z","votes":0,"approved":false,"apps":["discord","github"],"value":{"modules":[{"id":"a","value":{"lock":"","type":"rawscript","content":"import axios from \"npm:axios\";\nimport * as wmill from \"https://deno.land/x/windmill@v1.151.0/mod.ts\";\n\nconst GITHUB_API_URL = \"https://api.github.com/graphql\";\n\ntype Github = any;\n\nexport async function main(\n  githubAuth: Github,\n  minimalTimestamp: Date,\n  owner: string,\n  repository: string,\n) {\n  const state = await wmill.getState();\n\n  const timestamp = state ? new Date(state) : minimalTimestamp;\n  const nextTimestamp = new Date().toISOString()\n\n  const TOKEN = githubAuth.token;\n\n  async function fetchUpdatesSince(timestamp: Date) {\n    let updates: any[] = [];\n    let endCursor: string | null = null;\n\n    while (true) {\n      const response = await fetchDiscussionPage(endCursor);\n\n      if (response.data.errors) {\n        console.error(\"GraphQL Error:\", response.data.errors);\n        break;\n      }\n\n      const discussions = response.data.data.repository.discussions.edges;\n\n      let newerUpdates = discussions.filter((discussion: any) =>\n        new Date(discussion.node.updatedAt) > timestamp\n      );\n\n      if (newerUpdates.length === 0) {\n        break;\n      }\n\n      updates = updates.concat(newerUpdates.map((n) => ({\n        id: n.node.id,\n        title: n.node.title,\n        discussionNumber: n.node.number,\n        totalCount: n.node.comments.totalCount,\n      })));\n\n      const pageInfo = response.data.data.repository.discussions.pageInfo;\n      if (!pageInfo.hasNextPage) {\n        break;\n      }\n      endCursor = pageInfo.endCursor;\n    }\n\n    return updates;\n  }\n\n  async function fetchDiscussionPage(after: string | null) {\n    const afterClause = after ? `after: \"${after}\"` : \"\";\n    const query = `\n        query {\n            repository(owner: \"${owner}\", name: \"${repository}\") {\n                discussions(${afterClause}, last: 10, orderBy: {field: UPDATED_AT, direction: DESC}) {\n                                      pageInfo {\n                        endCursor\n                        hasNextPage\n                    }\n                    edges {\n                        node {\n                            id\n                            title\n                            body\n                            updatedAt\n                            number \n                            comments {\n                              totalCount\n                            } \n                                          \n                        }\n                        \n                    }                    \n                }\n            }\n        }\n    `;\n\n    return axios.post(GITHUB_API_URL, {\n      query,\n    }, {\n      headers: {\n        \"Authorization\": `Bearer ${TOKEN}`,\n      },\n    });\n  }\n\n  const updates = await fetchUpdatesSince(timestamp);\n\n  await wmill.setState(nextTimestamp)\n\n  return updates;\n}\n","language":"deno","input_transforms":{"owner":{"expr":"flow_input.owner","type":"javascript"},"githubAuth":{"expr":"flow_input.githubAuth","type":"javascript"},"repository":{"expr":"flow_input.repository","type":"javascript"},"minimalTimestamp":{"type":"static","value":"2023-08-18T06:00:00.000Z"}}},"summary":"Get updated discussions"},{"id":"b","value":{"type":"forloopflow","modules":[{"id":"c","value":{"lock":"","type":"rawscript","content":"import {\n  createClient,\n  SupabaseClient,\n} from \"https://esm.sh/@supabase/supabase-js@2.14.0\";\nimport axios from \"npm:axios\";\n\nconst GITHUB_API_URL = \"https://api.github.com/graphql\";\ntype Supabase = any;\ntype Github = {\n  token: string;\n};\n\ntype Discussion = {\n  \"id\": string;\n  \"title\": string;\n  \"totalCount\": number;\n  \"discussionNumber\": number;\n};\n\nexport async function main(\n  supabaseAuth: Supabase,\n  discussion: Discussion,\n  owner: string,\n  repository: string,\n  githubAuth: Github,\n) {\n  const supabase: SupabaseClient = createClient(\n    supabaseAuth.url,\n    supabaseAuth.key,\n  );\n\n  /**\n   * Need a table to store:\n   * - github_id: the discussion id\n   * - github_total_count: How many comments in that discussion that were already synced\n   * - discord_id: The thread id on Discord\n   * - discord_after_id: the most recent discord comment synced\n   */\n\n  const { data, error } = await supabase\n    .from(\"discord-github-sync\")\n    .select(\"*\")\n    .eq(\"github_id\", discussion.discussionNumber);\n\n  // TODO: Handle error\n\n  let discordId: string | undefined = undefined;\n  let totalCount: number = 0;\n\n  if (data && data[0]) {\n    discordId = data[0][\"discord_id\"];\n    totalCount = data[0][\"github_total_count\"];\n  } else {\n    // Should create a discord forum with title = discussion.title\n    discordId = \"THE_DISCORD_FORUM_ID_\" + discussion.id;\n  }\n\n  if (totalCount === discussion.totalCount) {\n\n    console.log('RETURNED')\n    // No comment to append\n    return;\n  }\n\n  async function fetchAllNewerComments(\n    discussionNumber: number,\n    count: number,\n  ): Promise<any[]> {\n    let comments: any[] = [];\n    let endCursor: string | null = null;\n\n    const batchSize = 100;\n    let tmpCount = count;\n\n    while (true) {\n      const response = await fetchCommentPage(\n        discussionNumber,\n        endCursor,\n        tmpCount <= batchSize ? tmpCount : batchSize,\n      );\n\n      if (tmpCount > batchSize) {\n        tmpCount = tmpCount - batchSize;\n      }\n\n      if (response.data.errors) {\n        break;\n      }\n\n      comments = comments.concat(\n        response.data.data.repository.discussion.comments\n          .nodes,\n      );\n\n      const pageInfo =\n        response.data.data.repository.discussion.comments.pageInfo;\n      if (!pageInfo.hasNextPage || tmpCount <= batchSize) {\n        break;\n      }\n      endCursor = pageInfo.endCursor;\n    }\n\n    return comments;\n  }\n\n  async function fetchCommentPage(\n    discussionNumber: number,\n    after: string | null,\n    count: number,\n  ) {\n    const afterClause = after ? `after: \"${after}\"` : \"\";\n    const query = `\n        query {\n            repository(owner: \"${owner}\", name: \"${repository}\") {\n                discussion(number: ${discussionNumber}) {\n                    comments(${afterClause}, last: ${count}) {\n                        pageInfo {\n                            endCursor\n                            hasNextPage\n                        }\n                        nodes {\n                            id\n                            body\n                            updatedAt\n                            createdAt\n                            \n                        }\n                    }\n                }\n            }\n        }\n    `;\n\n    return axios.post(GITHUB_API_URL, {\n      query,\n    }, {\n      headers: {\n        \"Authorization\": `Bearer ${githubAuth.token}`,\n      },\n    });\n  }\n\n  const newCommentCount = discussion.totalCount - totalCount;\n\n  const newerComments = await fetchAllNewerComments(\n    discussion.discussionNumber,\n    newCommentCount,\n  );\n\n  // TODO: Should post newerComments to Discord Forum using discordId and the discord api\n\n  if (data === null) {\n    const { error } = await supabase\n      .from(\"discord-github-sync\")\n      .insert({\n        github_id: discussion.discussionNumber,\n        discord_id: discordId,\n        github_total_count: discussion.totalCount,\n      });\n    // Handle error\n    console.log({ error });\n  } else {\n    const { error } = await supabase\n      .from(\"discord-github-sync\")\n      .update({\n        github_total_count: discussion.totalCount,\n      })\n      .eq(\"github_id\", discussion.discussionNumber);\n\n    // Handle error\n    console.log({ error });\n  }\n\n  return newerComments;\n}\n","language":"deno","input_transforms":{"owner":{"expr":"flow_input.owner","type":"javascript"},"discussion":{"expr":"flow_input.iter.value","type":"javascript"},"githubAuth":{"expr":"flow_input.githubAuth","type":"javascript"},"repository":{"expr":"flow_input.repository","type":"javascript"},"supabaseAuth":{"type":"static","value":"$res:u/faton/supabase-tws"}}},"summary":"Handle new comments"}],"iterator":{"expr":"results.a","type":"javascript"},"parallel":false,"skip_failures":true}}]},"schema":{"type":"object","$schema":"https://json-schema.org/draft/2020-12/schema","required":["githubAuth","owner","repository"],"properties":{"owner":{"type":"string","format":"","default":null,"description":""},"githubAuth":{"type":"object","format":"resource-github","default":null,"description":""},"repository":{"type":"string","format":"","default":null,"description":""}}},"description":"The flow manages GitHub discussion synchronization via a Supabase table. It fetches new comments, posts them on Discord, and updates records. The table stores discussion numbers, comment counts, Discord thread IDs, and the latest synced comment IDs, ensuring an organized flow of updates between GitHub and Discord.\n\nThe flow utilizes a Supabase table named \"discord-github-sync\" to manage synchronization information. The table has the following structure:\n\n- github_id: Stores the GitHub discussion number as a unique identifier.\n- github_total_count: Tracks the total number of comments in the GitHub discussion.\n- discord_id: Represents the Discord thread ID associated with the GitHub discussion.\n- discord_after_id: Keeps track of the most recent synced Discord comment.\n\nThe table is essential for tracking the synchronization status between GitHub and Discord. This enables efficient management of discussions, allowing the code to determine which discussions have been synchronized, which comments are new, and where to continue syncing from. \n\n\n\n\n\n","recording":null,"vcreated_at":"2023-08-18T14:00:13.166Z","vcreated_by":"rubenfiszel","comments":[]}}