build: turborepo monorepo

.gitignore

@@ -1,19 +1,20 @@
 # See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
 
 # dependencies
-/node_modules
+node_modules
 /.pnp
 .pnp.js
 
 # testing
 /coverage
 
 # next.js
-/.next/
+.next/
 /out/
 
 # production
-/build
+build
+dist
 
 # misc
 .DS_Store
@@ -31,6 +32,7 @@ yarn-error.log*
 
 # vercel
 .vercel
+.turbo
 
 tmp
 # Sentry Config File

.npmrc

@@ -0,0 +1 @@
+enable-pre-post-scripts=true

apps/backend/package.json

@@ -0,0 +1,53 @@
+{
+	"name": "backend",
+	"version": "1.0.0",
+	"description": "learnbyvideo.dev backend crawler",
+	"type": "module",
+	"main": "index.js",
+	"scripts": {
+		"deploy": "ssh node 'cd learnbyvideo.dev && npm run update:backend'",
+		"update": "git checkout backend && git reset --hard && git pull && npm install && npx prisma generate",
+		"find:videos": "node src/findVideos.js $1",
+		"find:videos:prod": "NODE_ENV=production node src/findVideos.js $1",
+		"find:videos:recheck": "node src/findVideos.js --find-new-videos false",
+		"find:videos:10": "node src/findVideos.js --limit 10",
+		"find:videos:prod:10": "NODE_ENV=production node src/findVideos.js --limit 10",
+		"find:videos:recheck:prod": "NODE_ENV=production node src/findVideos.js --find-new-videos false",
+		"update:videos": "node src/updateVideos.js $1",
+		"update:videos:prod": "NODE_ENV=production node src/updateVideos.js $1",
+		"update:channels:prod": "NODE_ENV=production node src/updateChannels.js $1",
+		"update:videos:recent": "node src/updateVideos.js -m 2",
+		"update:videos:recent:prod": "NODE_ENV=production node src/updateVideos.js -m 2",
+		"crawl:channels:prod": "NODE_ENV=production node src/crawlChannels.js $1",
+		"update:channels": "node src/updateChannels.js $1",
+		"crawl:channels": "node src/crawlChannels.js $1",
+		"add:channels": "node src/addChannel.js $1",
+		"quota": "node src/quota.js",
+		"quota:verbose": "node src/quota.js --verbose"
+	},
+	"dependencies": {
+		"@googleapis/youtube": "^13.0.0",
+		"config": "workspace:*",
+		"database": "workspace:*",
+		"date-fns": "^2.30.0",
+		"date-fns-tz": "^2.0.0",
+		"delay": "^6.0.0",
+		"dotenv-mono": "^1.3.13",
+		"lodash-es": "^4.17.21",
+		"minimost": "^1.2.0",
+		"node-fetch": "^3.3.2",
+		"picocolors": "^1.0.0",
+		"pluralize": "^8.0.0",
+		"pretty-ms": "^8.0.0",
+		"rss-parser": "^3.13.0",
+		"sparkline": "^0.2.0"
+	},
+	"keywords": [],
+	"author": "",
+	"license": "ISC",
+	"devDependencies": {
+		"autoprefixer": "^10.4.16",
+		"postcss": "^8.4.32",
+		"tailwindcss": "^3.4.0"
+	}
+}

backend/addChannel.mjs → apps/backend/src/addChannel.js

@@ -1,15 +1,14 @@
-#!/usr/bin/env node
-import 'dotenv/config';
+import 'dotenv-mono/load';
 
 import url from 'url';
 
 import minimost from 'minimost';
 
-import { addChannel } from './db.mjs';
-import { updateHomePage } from './lib.mjs';
-import { error, logMemoryUsage, logTimeSpent } from './util.mjs';
-import { youTubeVideosList } from './youtubeApi.mjs';
-import QuotaTracker from './youtubeQuota.mjs';
+import { addChannel } from './db.js';
+import { updateHomePage } from './lib.js';
+import { error, logMemoryUsage, logTimeSpent } from './util.js';
+import { youTubeVideosList } from './youtubeApi.js';
+import QuotaTracker from './youtubeQuota.js';
 
 const { flags: options, input: youtubeIds } = minimost(process.argv.slice(2), {
   string: ['status', 'default-category', 'type'],

backend/crawlChannels.mjs → apps/backend/src/crawlChannels.js

@@ -1,13 +1,13 @@
-import 'dotenv/config';
+import 'dotenv-mono/load';
 
-import { CrawlState } from '@prisma/client';
+import { CrawlState } from 'database';
 import minimost from 'minimost';
 import pluralize from 'pluralize';
 
-import { getChannels, updateChannel, upsertVideos } from './db.mjs';
-import { error, logMemoryUsage, logTimeSpent } from './util.mjs';
-import { crawlChannel } from './youtube.mjs';
-import QuotaTracker from './youtubeQuota.mjs';
+import { getChannels, updateChannel, upsertVideos } from './db.js';
+import { error, logMemoryUsage, logTimeSpent } from './util.js';
+import { crawlChannel } from './youtube.js';
+import QuotaTracker from './youtubeQuota.js';
 
 const options = minimost(process.argv.slice(2), {
   string: ['max-channels'],

backend/db.mjs → apps/backend/src/db.js

@@ -1,20 +1,19 @@
-import { CrawlState, VideoStatus, VideoType } from '@prisma/client';
+import { CrawlState, VideoStatus, VideoType, prisma } from 'database';
 import delay from 'delay';
 import { differenceBy, groupBy, map, merge, omit } from 'lodash-es';
 import pluralize from 'pluralize';
 
-import prisma from '../prisma/prisma.mjs';
-import { debug, error } from './util.mjs';
+import { debug, error } from './util.js';
 import {
   crawlChannel,
   extractChannelInfo,
   isShort,
   videoStatus,
   youtubeQuotaDate,
-} from './youtube.mjs';
-import { youTubeChannelsList } from './youtubeApi.mjs';
+} from './youtube.js';
+import { youTubeChannelsList } from './youtubeApi.js';
 
-import config from './config.mjs';
+import config from 'config';
 
 export const getChannels = (queryOptions) => prisma.channel.findMany(queryOptions);
 export const getChannel = (queryOptions) => prisma.channel.findUnique(queryOptions);

backend/findVideos.mjs → apps/backend/src/findVideos.js

@@ -1,5 +1,6 @@
-import 'dotenv/config';
+import 'dotenv-mono/load';
 
+import config from 'config';
 import delay from 'delay';
 import { map } from 'lodash-es';
 import minimost from 'minimost';
@@ -12,13 +13,11 @@ import {
   updateChannelMany,
   updateVideos,
   upsertVideos,
-} from './db.mjs';
-import { checkForDeletedVideos, updateHomePage } from './lib.mjs';
-import { error, logMemoryUsage, logTimeSpent, warn } from './util.mjs';
-import { getRecentVideosFromRSS, getVideoDetails } from './youtube.mjs';
-import QuotaTracker from './youtubeQuota.mjs';
-
-import config from './config.mjs';
+} from './db.js';
+import { checkForDeletedVideos, updateHomePage } from './lib.js';
+import { error, logMemoryUsage, logTimeSpent, warn } from './util.js';
+import { getRecentVideosFromRSS, getVideoDetails } from './youtube.js';
+import QuotaTracker from './youtubeQuota.js';
 
 const options = minimost(process.argv.slice(2), {
   string: ['channels', 'min-last-updated', 'max-last-updated'],

backend/lib.mjs → apps/backend/src/lib.js

@@ -1,11 +1,11 @@
-import { VideoStatus } from '@prisma/client';
+import { VideoStatus } from 'database';
 import { differenceBy } from 'lodash-es';
 import fetch from 'node-fetch';
 import pluralize from 'pluralize';
 
-import { updateVideo } from './db.mjs';
-import { error } from './util.mjs';
-import { missingVideoStatus, videoUrl } from './youtube.mjs';
+import { updateVideo } from './db.js';
+import { error } from './util.js';
+import { missingVideoStatus, videoUrl } from './youtube.js';
 
 // Rebuild the static homepage on Vercel via on demand ISR
 // eslint-disable-next-line import/prefer-default-export

backend/quota.mjs → apps/backend/src/quota.js

@@ -1,15 +1,14 @@
 #!/usr/bin/env node
-import 'dotenv/config';
+import 'dotenv-mono/load';
 
+import config from 'config';
+import { prisma } from 'database';
 import { map } from 'lodash-es';
 import minimost from 'minimost';
 import pc from 'picocolors';
 import sparkline from 'sparkline';
 
-import prisma from '../prisma/prisma.mjs';
-import QuotaTracker from './youtubeQuota.mjs';
-
-import config from './config.mjs';
+import QuotaTracker from './youtubeQuota.js';
 
 const options = minimost(process.argv.slice(2), {
   boolean: ['verbose'],

backend/updateChannels.mjs → apps/backend/src/updateChannels.js

@@ -1,13 +1,13 @@
-import 'dotenv/config';
+import 'dotenv-mono/load';
 
 import { map } from 'lodash-es';
 import pluralize from 'pluralize';
 
-import { getChannels, updateChannels } from './db.mjs';
-import { logMemoryUsage, logTimeSpent } from './util.mjs';
-import { extractChannelInfo } from './youtube.mjs';
-import { youTubeChannelsList } from './youtubeApi.mjs';
-import QuotaTracker from './youtubeQuota.mjs';
+import { getChannels, updateChannels } from './db.js';
+import { logMemoryUsage, logTimeSpent } from './util.js';
+import { extractChannelInfo } from './youtube.js';
+import { youTubeChannelsList } from './youtubeApi.js';
+import QuotaTracker from './youtubeQuota.js';
 
 (async () => {
   const startTime = Date.now();

backend/updateVideos.mjs → apps/backend/src/updateVideos.js

@@ -1,17 +1,16 @@
-import 'dotenv/config';
+import 'dotenv-mono/load';
 
-import { VideoStatus } from '@prisma/client';
+import config from 'config';
+import { VideoStatus } from 'database';
 import { uniqBy } from 'lodash-es';
 import minimost from 'minimost';
 import pluralize from 'pluralize';
 
-import { getRecheckVideos, getVideos, updateVideos } from './db.mjs';
-import { checkForDeletedVideos, updateHomePage } from './lib.mjs';
-import { debug, error, logMemoryUsage, logTimeSpent } from './util.mjs';
-import { getVideoDetails } from './youtube.mjs';
-import QuotaTracker from './youtubeQuota.mjs';
-
-import config from './config.mjs';
+import { getRecheckVideos, getVideos, updateVideos } from './db.js';
+import { checkForDeletedVideos, updateHomePage } from './lib.js';
+import { debug, error, logMemoryUsage, logTimeSpent } from './util.js';
+import { getVideoDetails } from './youtube.js';
+import QuotaTracker from './youtubeQuota.js';
 
 const options = minimost(process.argv.slice(2), {
   string: ['limit', 'offset', 'ids', 'min-last-published', 'order-by'],

backend/youtube.mjs → apps/backend/src/youtube.js

@@ -1,15 +1,13 @@
+import config from 'config';
+import { ChannelStatus, VideoStatus } from 'database';
 import { formatInTimeZone } from 'date-fns-tz';
 import { keyBy, map, uniqBy } from 'lodash-es';
 import fetch from 'node-fetch';
 import pluralize from 'pluralize';
 import Parser from 'rss-parser';
 
-import { ChannelStatus, VideoStatus } from '@prisma/client';
-
-import { error, warn } from './util.mjs';
-import { youTubePlaylistItems, youTubeVideosList } from './youtubeApi.mjs';
-
-import config from './config.mjs';
+import { error, warn } from './util.js';
+import { youTubePlaylistItems, youTubeVideosList } from './youtubeApi.js';
 
 const rssParser = new Parser({
   customFields: {

backend/youtubeApi.mjs → apps/backend/src/youtubeApi.js

@@ -1,9 +1,8 @@
 import { youtube } from '@googleapis/youtube';
+import config from 'config';
 import { chunk, merge } from 'lodash-es';
 
-import { debug } from './util.mjs';
-
-import config from './config.mjs';
+import { debug } from './util.js';
 
 const Youtube = youtube({
   version: 'v3',

backend/youtubeQuota.mjs → apps/backend/src/youtubeQuota.js

@@ -1,10 +1,9 @@
+import config from 'config';
 import pc from 'picocolors';
 import pluralize from 'pluralize';
 
-import { addQuotaUsage, todaysQuotaUsage } from './db.mjs';
-import { debug, error, warn } from './util.mjs';
-
-import config from './config.mjs';
+import { addQuotaUsage, todaysQuotaUsage } from './db.js';
+import { debug, error, warn } from './util.js';
 
 // Look Mom, I'm using classes in JavaScript!
 class QuotaTracker {

apps/site/jsconfig.json

@@ -0,0 +1,7 @@
+{
+  "compilerOptions": {
+    "paths": {
+      "@/*": ["./src/*"]
+    }
+  }
+}

next.config.mjs → apps/site/next.config.mjs

@@ -1,4 +1,5 @@
 import BundleAnalyzer from '@next/bundle-analyzer';
+import { PrismaPlugin } from '@prisma/nextjs-monorepo-workaround-plugin';
 import { withSentryConfig } from '@sentry/nextjs';
 
 const withBundleAnalyzer = BundleAnalyzer({
@@ -21,6 +22,13 @@ const nextConfig = {
     ],
     /* cspell:enable */
   },
+  webpack: (config, { isServer }) => {
+    if (isServer) {
+      config.plugins = [...config.plugins, new PrismaPlugin()];
+    }
+
+    return config;
+  },
 };
 
 export default withSentryConfig(

apps/site/package.json

@@ -0,0 +1,41 @@
+{
+	"name": "site",
+	"version": "1.0.0",
+	"description": "learnbyvideo.dev main site",
+	"main": "index.js",
+	"scripts": {
+		"dev": "next dev",
+		"build": "next build",
+		"build:prod": "NODE_ENV=production next build",
+		"start": "next start"
+	},
+	"dependencies": {
+		"@fontsource/inter": "^5.0.16",
+		"@fontsource/lato": "^5.0.18",
+		"@fontsource/roboto": "^5.0.8",
+		"@googleapis/youtube": "^13.0.0",
+		"@next/bundle-analyzer": "^14.0.4",
+		"@sentry/nextjs": "^7.91.0",
+		"@tanstack/react-query": "^5.15.0",
+		"@tanstack/react-query-devtools": "^5.15.0",
+		"@tanstack/react-query-next-experimental": "^5.15.0",
+		"@vercel/analytics": "^1.1.1",
+		"config": "workspace:*",
+		"database": "workspace:*",
+		"lodash-es": "^4.17.21",
+		"next": "14.0.4",
+		"pluralize": "^8.0.0",
+		"react": "18.2.0",
+		"react-dom": "18.2.0",
+		"react-infinite-scroller": "^1.2.6",
+		"react-spinners": "^0.13.8",
+		"react-timeago": "^7.2.0",
+		"server-only": "^0.0.1"
+	},
+	"keywords": [],
+	"author": "",
+	"license": "ISC",
+	"devDependencies": {
+		"@prisma/nextjs-monorepo-workaround-plugin": "^5.7.1"
+	}
+}

src/app/layout.js → apps/site/src/app/layout.js

@@ -1,11 +1,11 @@
 import { HydrationBoundary, QueryClient, dehydrate } from '@tanstack/react-query';
 
-import GoogleAnalytics from '@/components/GoogleAnalytics';
-import Providers from '@/components/Providers';
-import Footer from '@/components/ui/Footer';
-import Header from '@/components/ui/Header';
+import GoogleAnalytics from '../components/GoogleAnalytics';
+import Providers from '../components/Providers';
+import Footer from '../components/ui/Footer';
+import Header from '../components/ui/Header';
 
-import { getRecentVideos, getSiteData } from '@/lib/data';
+import { getRecentVideos, getSiteData } from '../lib/data';
 
 export default async function RootLayout({ children }) {
   const { videoCount, channelCount } = await getSiteData();