From 1b52812febe5c4e52cecdfe7a3c7e3332ee92ed5 Mon Sep 17 00:00:00 2001 From: yu-i-i Date: Tue, 3 Dec 2024 16:40:27 +0100 Subject: [PATCH 001/106] Redirect non-existing links to Overleaf page --- services/web/app/src/router.mjs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/services/web/app/src/router.mjs b/services/web/app/src/router.mjs index 10ed348a6c8..c17bfbb3785 100644 --- a/services/web/app/src/router.mjs +++ b/services/web/app/src/router.mjs @@ -1246,6 +1246,10 @@ async function initialize(webRouter, privateApiRouter, publicApiRouter) { ) } + webRouter.get(['/learn*', '/blog*', '/latex*', '/for/*', '/contact*'], (req, res) => { + res.redirect(301, `https://www.overleaf.com${req.originalUrl}`) + }) + webRouter.get('/unsupported-browser', renderUnsupportedBrowserPage) webRouter.get('*', ErrorController.notFound) From 60989f8645fba854d424b7e90c8631f3a8f49157 Mon Sep 17 00:00:00 2001 From: yu-i-i Date: Tue, 3 Dec 2024 01:07:49 +0100 Subject: [PATCH 002/106] Enable track changes and comments feature --- .../Features/Project/ProjectEditorHandler.mjs | 2 +- services/web/config/settings.defaults.js | 1 + .../hooks/use-review-panel-state.ts | 1659 +++++++++++++++++ .../app/src/TrackChangesController.js | 194 ++ .../app/src/TrackChangesRouter.js | 72 + services/web/modules/track-changes/index.js | 2 + 6 files changed, 1929 insertions(+), 1 deletion(-) create mode 100644 services/web/frontend/js/features/ide-react/context/review-panel/hooks/use-review-panel-state.ts create mode 100644 services/web/modules/track-changes/app/src/TrackChangesController.js create mode 100644 services/web/modules/track-changes/app/src/TrackChangesRouter.js create mode 100644 services/web/modules/track-changes/index.js diff --git a/services/web/app/src/Features/Project/ProjectEditorHandler.mjs b/services/web/app/src/Features/Project/ProjectEditorHandler.mjs index e6b0af442a1..7ad80fc6eb7 100644 --- a/services/web/app/src/Features/Project/ProjectEditorHandler.mjs +++ b/services/web/app/src/Features/Project/ProjectEditorHandler.mjs @@ -3,7 +3,7 @@ import Path from 'node:path' let ProjectEditorHandler export default ProjectEditorHandler = { - trackChangesAvailable: false, + trackChangesAvailable: true, buildProjectModelView( project, diff --git a/services/web/config/settings.defaults.js b/services/web/config/settings.defaults.js index eee3905ca2f..ee3025fa195 100644 --- a/services/web/config/settings.defaults.js +++ b/services/web/config/settings.defaults.js @@ -1067,6 +1067,7 @@ module.exports = { 'launchpad', 'server-ce-scripts', 'user-activate', + 'track-changes', ], viewIncludes: {}, diff --git a/services/web/frontend/js/features/ide-react/context/review-panel/hooks/use-review-panel-state.ts b/services/web/frontend/js/features/ide-react/context/review-panel/hooks/use-review-panel-state.ts new file mode 100644 index 00000000000..c712384163f --- /dev/null +++ b/services/web/frontend/js/features/ide-react/context/review-panel/hooks/use-review-panel-state.ts @@ -0,0 +1,1659 @@ +import { useState, useEffect, useMemo, useCallback, useRef } from 'react' +import { useTranslation } from 'react-i18next' +import { isEqual, cloneDeep } from 'lodash' +import usePersistedState from '@/shared/hooks/use-persisted-state' +import useScopeValue from '../../../../../shared/hooks/use-scope-value' +import useSocketListener from '@/features/ide-react/hooks/use-socket-listener' +import useAbortController from '@/shared/hooks/use-abort-controller' +import useScopeEventEmitter from '@/shared/hooks/use-scope-event-emitter' +import useLayoutToLeft from '@/features/ide-react/context/review-panel/hooks/useLayoutToLeft' +import { sendMB } from '@/infrastructure/event-tracking' +import { + dispatchReviewPanelLayout as handleLayoutChange, + UpdateType, +} from '@/features/source-editor/extensions/changes/change-manager' +import { useProjectContext } from '@/shared/context/project-context' +import { useLayoutContext } from '@/shared/context/layout-context' +import { useUserContext } from '@/shared/context/user-context' +import { useIdeReactContext } from '@/features/ide-react/context/ide-react-context' +import { useConnectionContext } from '@/features/ide-react/context/connection-context' +import { usePermissionsContext } from '@/features/ide-react/context/permissions-context' +import { useModalsContext } from '@/features/ide-react/context/modals-context' +import { + EditorManager, + useEditorManagerContext, +} from '@/features/ide-react/context/editor-manager-context' +import { debugConsole } from '@/utils/debugging' +import { deleteJSON, getJSON, postJSON } from '@/infrastructure/fetch-json' +import RangesTracker from '@overleaf/ranges-tracker' +import type * as ReviewPanel from '@/features/source-editor/context/review-panel/types/review-panel-state' +import { + CommentId, + ReviewPanelCommentThreadMessage, + ReviewPanelCommentThreads, + ReviewPanelDocEntries, + SubView, + ThreadId, +} from '../../../../../../../types/review-panel/review-panel' +import { UserId } from '../../../../../../../types/user' +import { PublicAccessLevel } from '../../../../../../../types/public-access-level' +import { + DeepReadonly, + Entries, + MergeAndOverride, +} from '../../../../../../../types/utils' +import { ReviewPanelCommentThread } from '../../../../../../../types/review-panel/comment-thread' +import { DocId } from '../../../../../../../types/project-settings' +import { + ReviewPanelAddCommentEntry, + ReviewPanelAggregateChangeEntry, + ReviewPanelBulkActionsEntry, + ReviewPanelChangeEntry, + ReviewPanelCommentEntry, + ReviewPanelEntry, +} from '../../../../../../../types/review-panel/entry' +import { + ReviewPanelCommentThreadMessageApi, + ReviewPanelCommentThreadsApi, +} from '../../../../../../../types/review-panel/api' +import { DateString } from '../../../../../../../types/helpers/date' +import { + Change, + CommentOperation, + EditOperation, +} from '../../../../../../../types/change' +import { RangesTrackerWithResolvedThreadIds } from '@/features/ide-react/editor/document-container' +import getMeta from '@/utils/meta' +import { useEditorContext } from '@/shared/context/editor-context' +import { getHueForUserId } from '@/shared/utils/colors' + +const dispatchReviewPanelEvent = (type: string, payload?: any) => { + window.dispatchEvent( + new CustomEvent('review-panel:event', { + detail: { type, payload }, + }) + ) +} + +const formatUser = (user: any): any => { + let isSelf, name + const id = + (user != null ? user._id : undefined) || + (user != null ? user.id : undefined) + + if (id == null) { + return { + id: 'anonymous-user', + email: null, + name: 'Anonymous', + isSelf: false, + hue: getHueForUserId(), + avatar_text: 'A', + } + } + if (id === getMeta('ol-user_id')) { + name = 'You' + isSelf = true + } else { + name = [user.first_name, user.last_name] + .filter(n => n != null && n !== '') + .join(' ') + if (name === '') { + name = + (user.email != null ? user.email.split('@')[0] : undefined) || 'Unknown' + } + isSelf = false + } + return { + id, + email: user.email, + name, + isSelf, + hue: getHueForUserId(id), + avatar_text: [user.first_name, user.last_name] + .filter(n => n != null) + .map(n => n[0]) + .join(''), + } +} + +const formatComment = ( + comment: ReviewPanelCommentThreadMessageApi +): ReviewPanelCommentThreadMessage => { + const commentTyped = comment as unknown as ReviewPanelCommentThreadMessage + commentTyped.user = formatUser(comment.user) + commentTyped.timestamp = new Date(comment.timestamp) + return commentTyped +} + +function useReviewPanelState(): ReviewPanel.ReviewPanelState { + const { t } = useTranslation() + const { reviewPanelOpen, setReviewPanelOpen, setMiniReviewPanelVisible } = + useLayoutContext() + const { projectId } = useIdeReactContext() + const project = useProjectContext() + const user = useUserContext() + const { socket } = useConnectionContext() + const { + features: { trackChangesVisible, trackChanges }, + } = project + const { isRestrictedTokenMember } = useEditorContext() + const { + openDocWithId, + currentDocument, + currentDocumentId, + wantTrackChanges, + setWantTrackChanges, + } = useEditorManagerContext() as MergeAndOverride< + EditorManager, + { currentDocumentId: DocId } + > + // TODO permissions to be removed from the review panel context. It currently acts just as a proxy. + const permissions = usePermissionsContext() + const { showGenericMessageModal } = useModalsContext() + const addCommentEmitter = useScopeEventEmitter('comment:start_adding') + + const layoutToLeft = useLayoutToLeft('.ide-react-editor-panel') + const [subView, setSubView] = + useState>('cur_file') + const [isOverviewLoading, setIsOverviewLoading] = + useState>(false) + // All selected changes. If an aggregated change (insertion + deletion) is selected, the two ids + // will be present. The length of this array will differ from the count below (see explanation). + const selectedEntryIds = useRef([]) + // A count of user-facing selected changes. An aggregated change (insertion + deletion) will count + // as only one. + const [nVisibleSelectedChanges, setNVisibleSelectedChanges] = + useState>(0) + const [collapsed, setCollapsed] = usePersistedState< + ReviewPanel.Value<'collapsed'> + >(`docs_collapsed_state:${projectId}`, {}, false, true) + const [commentThreads, setCommentThreads] = useState< + ReviewPanel.Value<'commentThreads'> + >({}) + const [entries, setEntries] = useState>({}) + const [users, setUsers] = useScopeValue>('users') + const [resolvedComments, setResolvedComments] = useState< + ReviewPanel.Value<'resolvedComments'> + >({}) + + const [shouldCollapse, setShouldCollapse] = + useState>(true) + const [lineHeight, setLineHeight] = + useState>(0) + + const [formattedProjectMembers, setFormattedProjectMembers] = useState< + ReviewPanel.Value<'formattedProjectMembers'> + >({}) + const [trackChangesState, setTrackChangesState] = useState< + ReviewPanel.Value<'trackChangesState'> + >({}) + const [trackChangesOnForEveryone, setTrackChangesOnForEveryone] = + useState>(false) + const [trackChangesOnForGuests, setTrackChangesOnForGuests] = + useState>(false) + const [trackChangesForGuestsAvailable, setTrackChangesForGuestsAvailable] = + useState>(false) + + const [resolvedThreadIds, setResolvedThreadIds] = useState< + Record + >({}) + + const [loadingThreads, setLoadingThreads] = + useScopeValue('loadingThreads') + + const loadThreadsController = useAbortController() + const threadsLoadedOnceRef = useRef(false) + const loadingThreadsInProgressRef = useRef(false) + const ensureThreadsAreLoaded = useCallback(() => { + if (threadsLoadedOnceRef.current) { + // We get any updates in real time so only need to load them once. + return + } + threadsLoadedOnceRef.current = true + loadingThreadsInProgressRef.current = true + + return getJSON(`/project/${projectId}/threads`, { + signal: loadThreadsController.signal, + }) + .then(threads => { + setLoadingThreads(false) + const tempResolvedThreadIds: typeof resolvedThreadIds = {} + const threadsEntries = Object.entries(threads) as [ + [ + ThreadId, + MergeAndOverride< + ReviewPanelCommentThread, + ReviewPanelCommentThreadsApi[ThreadId] + >, + ], + ] + for (const [threadId, thread] of threadsEntries) { + for (const comment of thread.messages) { + formatComment(comment) + } + if (thread.resolved_by_user) { + thread.resolved_by_user = formatUser(thread.resolved_by_user) + tempResolvedThreadIds[threadId] = true + } + } + setResolvedThreadIds(tempResolvedThreadIds) + setCommentThreads(threads as unknown as ReviewPanelCommentThreads) + + dispatchReviewPanelEvent('loaded_threads') + handleLayoutChange({ async: true }) + + return { + resolvedThreadIds: tempResolvedThreadIds, + commentThreads: threads, + } + }) + .catch(debugConsole.error) + .finally(() => { + loadingThreadsInProgressRef.current = false + }) + }, [loadThreadsController.signal, projectId, setLoadingThreads]) + + const rangesTrackers = useRef< + Record + >({}) + const refreshingRangeUsers = useRef(false) + const refreshedForUserIds = useRef(new Set()) + const refreshChangeUsers = useCallback( + (userId: UserId | null) => { + if (userId != null) { + if (refreshedForUserIds.current.has(userId)) { + // We've already tried to refresh to get this user id, so stop it looping + return + } + refreshedForUserIds.current.add(userId) + } + + // Only do one refresh at once + if (refreshingRangeUsers.current) { + return + } + refreshingRangeUsers.current = true + + getJSON(`/project/${projectId}/changes/users`) + .then(usersResponse => { + refreshingRangeUsers.current = false + const tempUsers = {} as ReviewPanel.Value<'users'> + // Always include ourself, since if we submit an op, we might need to display info + // about it locally before it has been flushed through the server + if (user) { + if (user.id) { + tempUsers[user.id] = formatUser(user) + } else { + tempUsers['anonymous-user'] = formatUser(user) + } + } + for (const user of usersResponse) { + if (user.id) { + tempUsers[user.id] = formatUser(user) + } else { + tempUsers['anonymous-user'] = formatUser(user) + } + } + setUsers(tempUsers) + }) + .catch(error => { + refreshingRangeUsers.current = false + debugConsole.error(error) + }) + }, + [projectId, setUsers, user] + ) + + const getChangeTracker = useCallback( + (docId: DocId) => { + if (!rangesTrackers.current[docId]) { + const rangesTracker = new RangesTracker([], []) + ;( + rangesTracker as RangesTrackerWithResolvedThreadIds + ).resolvedThreadIds = { ...resolvedThreadIds } + rangesTrackers.current[docId] = + rangesTracker as RangesTrackerWithResolvedThreadIds + } + return rangesTrackers.current[docId]! + }, + [resolvedThreadIds] + ) + + const getDocEntries = useCallback( + (docId: DocId) => { + return entries[docId] ?? ({} as ReviewPanelDocEntries) + }, + [entries] + ) + + const getDocResolvedComments = useCallback( + (docId: DocId) => { + return resolvedComments[docId] ?? ({} as ReviewPanelDocEntries) + }, + [resolvedComments] + ) + + const getThread = useCallback( + (threadId: ThreadId) => { + return ( + commentThreads[threadId] ?? + ({ messages: [] } as ReviewPanelCommentThread) + ) + }, + [commentThreads] + ) + + const updateEntries = useCallback( + async (docId: DocId) => { + const rangesTracker = getChangeTracker(docId) + const docEntries = cloneDeep(getDocEntries(docId)) + const docResolvedComments = cloneDeep(getDocResolvedComments(docId)) + // Assume we'll delete everything until we see it, then we'll remove it from this object + const deleteChanges = new Set() + + for (const [id, change] of Object.entries(docEntries) as Entries< + typeof docEntries + >) { + if ( + 'entry_ids' in change && + id !== 'add-comment' && + id !== 'bulk-actions' + ) { + for (const entryId of change.entry_ids) { + deleteChanges.add(entryId) + } + } + } + for (const [, change] of Object.entries(docResolvedComments) as Entries< + typeof docResolvedComments + >) { + if ('entry_ids' in change) { + for (const entryId of change.entry_ids) { + deleteChanges.add(entryId) + } + } + } + + let potentialAggregate = false + let prevInsertion = null + + for (const change of rangesTracker.changes as any[]) { + if ( + potentialAggregate && + change.op.d && + change.op.p === prevInsertion.op.p + prevInsertion.op.i.length && + change.metadata.user_id === prevInsertion.metadata.user_id + ) { + // An actual aggregate op. + const aggregateChangeEntries = docEntries as Record< + string, + ReviewPanelAggregateChangeEntry + > + aggregateChangeEntries[prevInsertion.id].type = 'aggregate-change' + aggregateChangeEntries[prevInsertion.id].metadata.replaced_content = + change.op.d + aggregateChangeEntries[prevInsertion.id].entry_ids.push(change.id) + } else { + if (docEntries[change.id] == null) { + docEntries[change.id] = {} as ReviewPanelEntry + } + deleteChanges.delete(change.id) + const newEntry: Partial = { + type: change.op.i ? 'insert' : 'delete', + entry_ids: [change.id], + content: change.op.i || change.op.d, + offset: change.op.p, + metadata: change.metadata, + } + for (const [key, value] of Object.entries(newEntry) as Entries< + typeof newEntry + >) { + const entriesTyped = docEntries[change.id] as Record + entriesTyped[key] = value + } + } + + if (change.op.i) { + potentialAggregate = true + prevInsertion = change + } else { + potentialAggregate = false + prevInsertion = null + } + + if (!users[change.metadata.user_id]) { + if (!isRestrictedTokenMember) { + refreshChangeUsers(change.metadata.user_id) + } + } + } + + let localResolvedThreadIds = resolvedThreadIds + + if (!isRestrictedTokenMember && rangesTracker.comments.length > 0) { + const threadsLoadResult = await ensureThreadsAreLoaded() + if (threadsLoadResult?.resolvedThreadIds) { + localResolvedThreadIds = threadsLoadResult.resolvedThreadIds + } + } else if (loadingThreads) { + // ensure that tracked changes are highlighted even if no comments are loaded + setLoadingThreads(false) + dispatchReviewPanelEvent('loaded_threads') + } + + if (!loadingThreadsInProgressRef.current) { + for (const comment of rangesTracker.comments) { + const commentId = comment.id as ThreadId + deleteChanges.delete(commentId) + + let newComment: any + if (localResolvedThreadIds[comment.op.t]) { + docResolvedComments[commentId] ??= {} as ReviewPanelCommentEntry + newComment = docResolvedComments[commentId] + delete docEntries[commentId] + } else { + docEntries[commentId] ??= {} as ReviewPanelEntry + newComment = docEntries[commentId] + delete docResolvedComments[commentId] + } + + newComment.type = 'comment' + newComment.thread_id = comment.op.t + newComment.entry_ids = [comment.id] + newComment.content = comment.op.c + newComment.offset = comment.op.p + } + } + + deleteChanges.forEach(changeId => { + delete docEntries[changeId] + delete docResolvedComments[changeId] + }) + + setEntries(prev => { + return isEqual(prev[docId], docEntries) + ? prev + : { ...prev, [docId]: docEntries } + }) + setResolvedComments(prev => { + return isEqual(prev[docId], docResolvedComments) + ? prev + : { ...prev, [docId]: docResolvedComments } + }) + + return docEntries + }, + [ + getChangeTracker, + getDocEntries, + getDocResolvedComments, + refreshChangeUsers, + resolvedThreadIds, + users, + ensureThreadsAreLoaded, + loadingThreads, + setLoadingThreads, + isRestrictedTokenMember, + ] + ) + + const regenerateTrackChangesId = useCallback( + (doc: typeof currentDocument) => { + if (doc) { + const currentChangeTracker = getChangeTracker(doc.doc_id as DocId) + const oldId = currentChangeTracker.getIdSeed() + const newId = RangesTracker.generateIdSeed() + currentChangeTracker.setIdSeed(newId) + doc.setTrackChangesIdSeeds({ pending: newId, inflight: oldId }) + } + }, + [getChangeTracker] + ) + + useEffect(() => { + if (!currentDocument) { + return + } + // The open doc range tracker is kept up to date in real-time so + // replace any outdated info with this + const rangesTracker = currentDocument.ranges! + ;(rangesTracker as RangesTrackerWithResolvedThreadIds).resolvedThreadIds = { + ...resolvedThreadIds, + } + rangesTrackers.current[currentDocument.doc_id as DocId] = + rangesTracker as RangesTrackerWithResolvedThreadIds + currentDocument.on('flipped_pending_to_inflight', () => + regenerateTrackChangesId(currentDocument) + ) + regenerateTrackChangesId(currentDocument) + + return () => { + currentDocument.off('flipped_pending_to_inflight') + } + }, [currentDocument, regenerateTrackChangesId, resolvedThreadIds]) + + const currentUserType = useCallback((): 'member' | 'guest' | 'anonymous-user' => { + if (!user) { + return 'anonymous-user' + } + if (project.owner._id === user.id) { + return 'member' + } + for (const member of project.members as any[]) { + if (member._id === user.id) { + return 'member' + } + } + return 'guest' + }, [project.members, project.owner, user]) + + const applyClientTrackChangesStateToServer = useCallback( + ( + trackChangesOnForEveryone: boolean, + trackChangesOnForGuests: boolean, + trackChangesState: ReviewPanel.Value<'trackChangesState'> + ) => { + const data: { + on?: boolean + on_for?: Record + on_for_guests?: boolean + } = {} + if (trackChangesOnForEveryone) { + data.on = true + } else { + data.on_for = {} + const entries = Object.entries(trackChangesState) as Array< + [ + UserId, + NonNullable< + (typeof trackChangesState)[keyof typeof trackChangesState] + >, + ] + > + for (const [userId, { value }] of entries) { + data.on_for[userId] = value + } + if (trackChangesOnForGuests) { + data.on_for_guests = true + } + } + postJSON(`/project/${projectId}/track_changes`, { + body: data, + }).catch(debugConsole.error) + }, + [projectId] + ) + + const setGuestsTCState = useCallback( + (newValue: boolean) => { + setTrackChangesOnForGuests(newValue) + if (currentUserType() === 'guest' || currentUserType() === 'anonymous-user') { + setWantTrackChanges(newValue) + } + }, + [currentUserType, setWantTrackChanges] + ) + + const setUserTCState = useCallback( + ( + trackChangesState: DeepReadonly>, + userId: UserId, + newValue: boolean, + isLocal = false + ) => { + const newTrackChangesState: ReviewPanel.Value<'trackChangesState'> = { + ...trackChangesState, + } + const state = + newTrackChangesState[userId] ?? + ({} as NonNullable<(typeof newTrackChangesState)[UserId]>) + newTrackChangesState[userId] = state + + if (state.syncState == null || state.syncState === 'synced') { + state.value = newValue + state.syncState = 'synced' + } else if (state.syncState === 'pending' && state.value === newValue) { + state.syncState = 'synced' + } else if (isLocal) { + state.value = newValue + state.syncState = 'pending' + } + + setTrackChangesState(newTrackChangesState) + + if (userId === user.id) { + setWantTrackChanges(newValue) + } + + return newTrackChangesState + }, + [setWantTrackChanges, user.id] + ) + + const setEveryoneTCState = useCallback( + (newValue: boolean, isLocal = false) => { + setTrackChangesOnForEveryone(newValue) + let newTrackChangesState: ReviewPanel.Value<'trackChangesState'> = { + ...trackChangesState, + } + for (const member of project.members as any[]) { + newTrackChangesState = setUserTCState( + newTrackChangesState, + member._id, + newValue, + isLocal + ) + } + setGuestsTCState(newValue) + + newTrackChangesState = setUserTCState( + newTrackChangesState, + project.owner._id, + newValue, + isLocal + ) + + return { trackChangesState: newTrackChangesState } + }, + [ + project.members, + project.owner._id, + setGuestsTCState, + setUserTCState, + trackChangesState, + ] + ) + + const toggleTrackChangesForEveryone = useCallback< + ReviewPanel.UpdaterFn<'toggleTrackChangesForEveryone'> + >( + (onForEveryone: boolean) => { + const { trackChangesState } = setEveryoneTCState(onForEveryone, true) + setGuestsTCState(onForEveryone) + applyClientTrackChangesStateToServer( + onForEveryone, + onForEveryone, + trackChangesState + ) + }, + [applyClientTrackChangesStateToServer, setEveryoneTCState, setGuestsTCState] + ) + + const toggleTrackChangesForGuests = useCallback< + ReviewPanel.UpdaterFn<'toggleTrackChangesForGuests'> + >( + (onForGuests: boolean) => { + setGuestsTCState(onForGuests) + applyClientTrackChangesStateToServer( + trackChangesOnForEveryone, + onForGuests, + trackChangesState + ) + }, + [ + applyClientTrackChangesStateToServer, + setGuestsTCState, + trackChangesOnForEveryone, + trackChangesState, + ] + ) + + const toggleTrackChangesForUser = useCallback< + ReviewPanel.UpdaterFn<'toggleTrackChangesForUser'> + >( + (onForUser: boolean, userId: UserId) => { + const newTrackChangesState = setUserTCState( + trackChangesState, + userId, + onForUser, + true + ) + applyClientTrackChangesStateToServer( + trackChangesOnForEveryone, + trackChangesOnForGuests, + newTrackChangesState + ) + }, + [ + applyClientTrackChangesStateToServer, + setUserTCState, + trackChangesOnForEveryone, + trackChangesOnForGuests, + trackChangesState, + ] + ) + + const applyTrackChangesStateToClient = useCallback( + (state: boolean | Record) => { + if (typeof state === 'boolean') { + setEveryoneTCState(state) + setGuestsTCState(state) + } else { + setTrackChangesOnForEveryone(false) + // TODO + // @ts-ignore + setGuestsTCState(state.__guests__ === true) + + let newTrackChangesState: ReviewPanel.Value<'trackChangesState'> = { + ...trackChangesState, + } + for (const member of project.members as any[]) { + newTrackChangesState = setUserTCState( + newTrackChangesState, + member._id, + !!state[member._id] + ) + } + newTrackChangesState = setUserTCState( + newTrackChangesState, + project.owner._id, + !!state[project.owner._id] + ) + return newTrackChangesState + } + }, + [ + project.members, + project.owner._id, + setEveryoneTCState, + setGuestsTCState, + setUserTCState, + trackChangesState, + ] + ) + + const setGuestFeatureBasedOnProjectAccessLevel = ( + projectPublicAccessLevel?: PublicAccessLevel + ) => { + setTrackChangesForGuestsAvailable(projectPublicAccessLevel === 'tokenBased') + } + + useEffect(() => { + setGuestFeatureBasedOnProjectAccessLevel(project.publicAccessLevel) + }, [project.publicAccessLevel]) + + useEffect(() => { + if ( + trackChangesForGuestsAvailable || + !trackChangesOnForGuests || + trackChangesOnForEveryone + ) { + return + } + + // Overrides guest setting + toggleTrackChangesForGuests(false) + }, [ + toggleTrackChangesForGuests, + trackChangesForGuestsAvailable, + trackChangesOnForEveryone, + trackChangesOnForGuests, + ]) + + const projectJoinedEffectExecuted = useRef(false) + useEffect(() => { + if (!projectJoinedEffectExecuted.current) { + projectJoinedEffectExecuted.current = true + requestAnimationFrame(() => { + if (trackChanges) { + applyTrackChangesStateToClient(project.trackChangesState) + } else { + applyTrackChangesStateToClient(false) + } + setGuestFeatureBasedOnProjectAccessLevel(project.publicAccessLevel) + }) + } + }, [ + applyTrackChangesStateToClient, + trackChanges, + project.publicAccessLevel, + project.trackChangesState, + ]) + + useEffect(() => { + setFormattedProjectMembers(prevState => { + const tempFormattedProjectMembers: typeof prevState = {} + if (project.owner) { + tempFormattedProjectMembers[project.owner._id] = formatUser( + project.owner + ) + } + const members = project.members ?? [] + for (const member of members) { + if (member.privileges === 'readAndWrite') { + if (!trackChangesState[member._id]) { + // An added member will have track changes enabled if track changes is on for everyone + setUserTCState( + trackChangesState, + member._id, + trackChangesOnForEveryone, + true + ) + } + tempFormattedProjectMembers[member._id] = formatUser(member) + } + } + return tempFormattedProjectMembers + }) + }, [ + project.members, + project.owner, + setUserTCState, + trackChangesOnForEveryone, + trackChangesState, + ]) + + useSocketListener( + socket, + 'toggle-track-changes', + applyTrackChangesStateToClient + ) + + const gotoEntry = useCallback( + (docId: DocId, entryOffset: number) => { + openDocWithId(docId, { gotoOffset: entryOffset }) + }, + [openDocWithId] + ) + + const view = reviewPanelOpen ? subView : 'mini' + + const toggleReviewPanel = useCallback(() => { + if (!trackChangesVisible) { + return + } + setReviewPanelOpen(!reviewPanelOpen) + sendMB('rp-toggle-panel', { + value: !reviewPanelOpen, + }) + }, [reviewPanelOpen, setReviewPanelOpen, trackChangesVisible]) + + const onCommentResolved = useCallback( + (threadId: ThreadId, user: any) => { + setCommentThreads(prevState => { + const thread = { ...getThread(threadId) } + thread.resolved = true + thread.resolved_by_user = formatUser(user) + thread.resolved_at = new Date().toISOString() as DateString + return { ...prevState, [threadId]: thread } + }) + setResolvedThreadIds(prevState => ({ ...prevState, [threadId]: true })) + setTimeout(() => { + dispatchReviewPanelEvent('comment:resolve_threads', [threadId]) + }) + }, + [getThread] + ) + + const resolveComment = useCallback( + (docId: DocId, entryId: ThreadId) => { + const docEntries = getDocEntries(docId) + const entry = docEntries[entryId] as ReviewPanelCommentEntry + + setEntries(prevState => ({ + ...prevState, + [docId]: { + ...prevState[docId], + [entryId]: { + ...prevState[docId][entryId], + focused: false, + }, + }, + })) + + postJSON( + `/project/${projectId}/doc/${docId}/thread/${entry.thread_id}/resolve` + ) + onCommentResolved(entry.thread_id, user) + sendMB('rp-comment-resolve', { view }) + }, + [getDocEntries, onCommentResolved, projectId, user, view] + ) + + const onCommentReopened = useCallback( + (threadId: ThreadId) => { + setCommentThreads(prevState => { + const { + resolved: _1, + resolved_by_user: _2, + resolved_at: _3, + ...thread + } = getThread(threadId) + return { ...prevState, [threadId]: thread } + }) + setResolvedThreadIds(({ [threadId]: _, ...resolvedThreadIds }) => { + return resolvedThreadIds + }) + setTimeout(() => { + dispatchReviewPanelEvent('comment:unresolve_thread', threadId) + }) + }, + [getThread] + ) + + const unresolveComment = useCallback( + (docId: DocId, threadId: ThreadId) => { + onCommentReopened(threadId) + const url = `/project/${projectId}/doc/${docId}/thread/${threadId}/reopen` + postJSON(url).catch(debugConsole.error) + sendMB('rp-comment-reopen') + }, + [onCommentReopened, projectId] + ) + + const onThreadDeleted = useCallback((threadId: ThreadId) => { + setResolvedThreadIds(({ [threadId]: _, ...resolvedThreadIds }) => { + return resolvedThreadIds + }) + setCommentThreads(({ [threadId]: _, ...commentThreads }) => { + return commentThreads + }) + dispatchReviewPanelEvent('comment:remove', threadId) + }, []) + + const deleteThread = useCallback( + (docId: DocId, threadId: ThreadId) => { + onThreadDeleted(threadId) + deleteJSON(`/project/${projectId}/doc/${docId}/thread/${threadId}`).catch( + debugConsole.error + ) + sendMB('rp-comment-delete') + }, + [onThreadDeleted, projectId] + ) + + const onCommentEdited = useCallback( + (threadId: ThreadId, commentId: CommentId, content: string) => { + setCommentThreads(prevState => { + const thread = { ...getThread(threadId) } + thread.messages = thread.messages.map(message => { + return message.id === commentId ? { ...message, content } : message + }) + return { ...prevState, [threadId]: thread } + }) + }, + [getThread] + ) + + const saveEdit = useCallback( + (threadId: ThreadId, commentId: CommentId, content: string) => { + const url = `/project/${projectId}/thread/${threadId}/messages/${commentId}/edit` + postJSON(url, { body: { content } }).catch(debugConsole.error) + handleLayoutChange({ async: true }) + }, + [projectId] + ) + + const onCommentDeleted = useCallback( + (threadId: ThreadId, commentId: CommentId) => { + setCommentThreads(prevState => { + const thread = { ...getThread(threadId) } + thread.messages = thread.messages.filter(m => m.id !== commentId) + return { ...prevState, [threadId]: thread } + }) + }, + [getThread] + ) + + const deleteComment = useCallback( + (threadId: ThreadId, commentId: CommentId) => { + onCommentDeleted(threadId, commentId) + deleteJSON( + `/project/${projectId}/thread/${threadId}/messages/${commentId}` + ).catch(debugConsole.error) + handleLayoutChange({ async: true }) + }, + [onCommentDeleted, projectId] + ) + + const doAcceptChanges = useCallback( + (entryIds: ThreadId[]) => { + const url = `/project/${projectId}/doc/${currentDocumentId}/changes/accept` + postJSON(url, { body: { change_ids: entryIds } }).catch( + debugConsole.error + ) + dispatchReviewPanelEvent('changes:accept', entryIds) + }, + [currentDocumentId, projectId] + ) + + const acceptChanges = useCallback( + (entryIds: ThreadId[]) => { + doAcceptChanges(entryIds) + sendMB('rp-changes-accepted', { view }) + }, + [doAcceptChanges, view] + ) + + const doRejectChanges = useCallback((entryIds: ThreadId[]) => { + dispatchReviewPanelEvent('changes:reject', entryIds) + }, []) + + const rejectChanges = useCallback( + (entryIds: ThreadId[]) => { + doRejectChanges(entryIds) + sendMB('rp-changes-rejected', { view }) + }, + [doRejectChanges, view] + ) + + const bulkAcceptActions = useCallback(() => { + doAcceptChanges(selectedEntryIds.current) + sendMB('rp-bulk-accept', { view, nEntries: nVisibleSelectedChanges }) + }, [doAcceptChanges, nVisibleSelectedChanges, view]) + + const bulkRejectActions = useCallback(() => { + doRejectChanges(selectedEntryIds.current) + sendMB('rp-bulk-reject', { view, nEntries: nVisibleSelectedChanges }) + }, [doRejectChanges, nVisibleSelectedChanges, view]) + + const refreshRanges = useCallback(() => { + type Doc = { + id: DocId + ranges: { + comments?: Change[] + changes?: Change[] + } + } + + return getJSON(`/project/${projectId}/ranges`) + .then(docs => { + setCollapsed(prevState => { + const collapsed = { ...prevState } + docs.forEach(doc => { + if (collapsed[doc.id] == null) { + collapsed[doc.id] = false + } + }) + return collapsed + }) + + docs.forEach(async doc => { + if (doc.id !== currentDocumentId) { + // this is kept up to date in real-time, don't overwrite + const rangesTracker = getChangeTracker(doc.id) + rangesTracker.comments = doc.ranges?.comments ?? [] + rangesTracker.changes = doc.ranges?.changes ?? [] + } + }) + + return Promise.all(docs.map(doc => updateEntries(doc.id))) + }) + .catch(debugConsole.error) + }, [ + currentDocumentId, + getChangeTracker, + projectId, + setCollapsed, + updateEntries, + ]) + + const handleSetSubview = useCallback((subView: SubView) => { + setSubView(subView) + sendMB('rp-subview-change', { subView }) + }, []) + + const submitReply = useCallback( + (threadId: ThreadId, replyContent: string) => { + const url = `/project/${projectId}/thread/${threadId}/messages` + postJSON(url, { body: { content: replyContent } }).catch(() => { + showGenericMessageModal( + t('error_submitting_comment'), + t('comment_submit_error') + ) + }) + + const trackingMetadata = { + view, + size: replyContent.length, + thread: threadId, + } + + setCommentThreads(prevState => ({ + ...prevState, + [threadId]: { ...getThread(threadId), submitting: true }, + })) + handleLayoutChange({ async: true }) + sendMB('rp-comment-reply', trackingMetadata) + }, + [getThread, projectId, showGenericMessageModal, t, view] + ) + + // TODO `submitNewComment` is partially localized in the `add-comment-entry` component. + const submitNewComment = useCallback( + (content: string) => { + if (!content) { + return + } + + const entries = getDocEntries(currentDocumentId) + const addCommentEntry = entries['add-comment'] as + | ReviewPanelAddCommentEntry + | undefined + + if (!addCommentEntry) { + return + } + + const { offset, length } = addCommentEntry + const threadId = RangesTracker.generateId() as ThreadId + setCommentThreads(prevState => ({ + ...prevState, + [threadId]: { ...getThread(threadId), submitting: true }, + })) + + const url = `/project/${projectId}/thread/${threadId}/messages` + postJSON(url, { body: { content } }) + .then(() => { + dispatchReviewPanelEvent('comment:add', { threadId, offset, length }) + handleLayoutChange({ async: true }) + sendMB('rp-new-comment', { size: content.length }) + }) + .catch(() => { + showGenericMessageModal( + t('error_submitting_comment'), + t('comment_submit_error') + ) + }) + }, + [ + currentDocumentId, + getDocEntries, + getThread, + projectId, + showGenericMessageModal, + t, + ] + ) + + const [isAddingComment, setIsAddingComment] = useState(false) + const [navHeight, setNavHeight] = useState(0) + const [toolbarHeight, setToolbarHeight] = useState(0) + const [layoutSuspended, setLayoutSuspended] = useState(false) + const [unsavedComment, setUnsavedComment] = useState('') + + useEffect(() => { + if (!trackChangesVisible) { + setReviewPanelOpen(false) + } + }, [trackChangesVisible, setReviewPanelOpen]) + + const hasEntries = useMemo(() => { + const docEntries = getDocEntries(currentDocumentId) + const permEntriesCount = Object.keys(docEntries).filter(key => { + return !['add-comment', 'bulk-actions'].includes(key) + }).length + return permEntriesCount > 0 && trackChangesVisible + }, [currentDocumentId, getDocEntries, trackChangesVisible]) + + useEffect(() => { + setMiniReviewPanelVisible(!reviewPanelOpen && !!hasEntries) + }, [reviewPanelOpen, hasEntries, setMiniReviewPanelVisible]) + + // listen for events from the CodeMirror 6 track changes extension + useEffect(() => { + const toggleTrackChangesFromKbdShortcut = () => { + const userId = user.id + if (trackChangesVisible && trackChanges && userId) { + const state = trackChangesState[userId] + if (state) { + toggleTrackChangesForUser(!state.value, userId) + } + } + } + + const editorLineHeightChanged = (payload: typeof lineHeight) => { + setLineHeight(payload) + handleLayoutChange() + } + + const editorTrackChangesChanged = async () => { + const tempEntries = cloneDeep(await updateEntries(currentDocumentId)) + + // `tempEntries` would be mutated + dispatchReviewPanelEvent('recalculate-screen-positions', { + entries: tempEntries, + updateType: 'trackedChangesChange', + }) + + // The state should be updated after dispatching the 'recalculate-screen-positions' + // event as `tempEntries` will be mutated + setEntries(prev => ({ ...prev, [currentDocumentId]: tempEntries })) + handleLayoutChange() + } + + const editorTrackChangesVisibilityChanged = () => { + handleLayoutChange({ async: true, animate: false }) + } + + const editorFocusChanged = ( + selectionOffsetStart: number, + selectionOffsetEnd: number, + selection: boolean, + updateType: UpdateType + ) => { + let tempEntries = cloneDeep(getDocEntries(currentDocumentId)) + // All selected changes will be added to this array. + selectedEntryIds.current = [] + // Count of user-visible changes, i.e. an aggregated change will count as one. + let tempNVisibleSelectedChanges = 0 + + const offset = selectionOffsetStart + const length = selectionOffsetEnd - selectionOffsetStart + + // Recreate the add comment and bulk actions entries only when + // necessary. This is to avoid the UI thinking that these entries have + // changed and getting into an infinite loop. + if (selection) { + const existingAddComment = tempEntries[ + 'add-comment' + ] as ReviewPanelAddCommentEntry + if ( + !existingAddComment || + existingAddComment.offset !== offset || + existingAddComment.length !== length + ) { + tempEntries['add-comment'] = { + type: 'add-comment', + offset, + length, + } as ReviewPanelAddCommentEntry + } + const existingBulkActions = tempEntries[ + 'bulk-actions' + ] as ReviewPanelBulkActionsEntry + if ( + !existingBulkActions || + existingBulkActions.offset !== offset || + existingBulkActions.length !== length + ) { + tempEntries['bulk-actions'] = { + type: 'bulk-actions', + offset, + length, + } as ReviewPanelBulkActionsEntry + } + } else { + delete (tempEntries as Partial)['add-comment'] + delete (tempEntries as Partial)['bulk-actions'] + } + + for (const [key, entry] of Object.entries(tempEntries) as Entries< + typeof tempEntries + >) { + let isChangeEntryAndWithinSelection = false + if (entry.type === 'comment' && !resolvedThreadIds[entry.thread_id]) { + tempEntries = { + ...tempEntries, + [key]: { + ...tempEntries[key], + focused: + entry.offset <= selectionOffsetStart && + selectionOffsetStart <= entry.offset + entry.content.length, + }, + } + } else if ( + entry.type === 'insert' || + entry.type === 'aggregate-change' + ) { + isChangeEntryAndWithinSelection = + entry.offset >= selectionOffsetStart && + entry.offset + entry.content.length <= selectionOffsetEnd + tempEntries = { + ...tempEntries, + [key]: { + ...tempEntries[key], + focused: + entry.offset <= selectionOffsetStart && + selectionOffsetStart <= entry.offset + entry.content.length, + }, + } + } else if (entry.type === 'delete') { + isChangeEntryAndWithinSelection = + selectionOffsetStart <= entry.offset && + entry.offset <= selectionOffsetEnd + tempEntries = { + ...tempEntries, + [key]: { + ...tempEntries[key], + focused: entry.offset === selectionOffsetStart, + }, + } + } else if ( + ['add-comment', 'bulk-actions'].includes(entry.type) && + selection + ) { + tempEntries = { + ...tempEntries, + [key]: { ...tempEntries[key], focused: true }, + } + } + if (isChangeEntryAndWithinSelection) { + const entryIds = 'entry_ids' in entry ? entry.entry_ids : [] + for (const entryId of entryIds) { + selectedEntryIds.current.push(entryId) + } + tempNVisibleSelectedChanges++ + } + } + + // `tempEntries` would be mutated + dispatchReviewPanelEvent('recalculate-screen-positions', { + entries: tempEntries, + updateType, + }) + + // The state should be updated after dispatching the 'recalculate-screen-positions' + // event as `tempEntries` will be mutated + setEntries(prev => ({ ...prev, [currentDocumentId]: tempEntries })) + setNVisibleSelectedChanges(tempNVisibleSelectedChanges) + + handleLayoutChange() + } + + const addNewCommentFromKbdShortcut = () => { + if (!trackChangesVisible) { + return + } + dispatchReviewPanelEvent('comment:select_line') + + if (!reviewPanelOpen) { + toggleReviewPanel() + } + handleLayoutChange({ async: true }) + addCommentEmitter() + } + + const handleEditorEvents = (e: Event) => { + const event = e as CustomEvent + const { type, payload } = event.detail + + switch (type) { + case 'line-height': { + editorLineHeightChanged(payload) + break + } + + case 'track-changes:changed': { + editorTrackChangesChanged() + break + } + + case 'track-changes:visibility_changed': { + editorTrackChangesVisibilityChanged() + break + } + + case 'focus:changed': { + const { from, to, empty, updateType } = payload + editorFocusChanged(from, to, !empty, updateType) + break + } + + case 'add-new-comment': { + addNewCommentFromKbdShortcut() + break + } + + case 'toggle-track-changes': { + toggleTrackChangesFromKbdShortcut() + break + } + + case 'toggle-review-panel': { + toggleReviewPanel() + break + } + } + } + + window.addEventListener('editor:event', handleEditorEvents) + + return () => { + window.removeEventListener('editor:event', handleEditorEvents) + } + }, [ + addCommentEmitter, + currentDocumentId, + getDocEntries, + resolvedThreadIds, + reviewPanelOpen, + toggleReviewPanel, + toggleTrackChangesForUser, + trackChanges, + trackChangesState, + trackChangesVisible, + updateEntries, + user.id, + ]) + + useSocketListener(socket, 'reopen-thread', onCommentReopened) + useSocketListener(socket, 'delete-thread', onThreadDeleted) + useSocketListener(socket, 'resolve-thread', onCommentResolved) + useSocketListener(socket, 'edit-message', onCommentEdited) + useSocketListener(socket, 'delete-message', onCommentDeleted) + useSocketListener( + socket, + 'accept-changes', + useCallback( + (docId: DocId, entryIds: ThreadId[]) => { + if (docId !== currentDocumentId) { + getChangeTracker(docId).removeChangeIds(entryIds) + } else { + dispatchReviewPanelEvent('changes:accept', entryIds) + } + updateEntries(docId) + }, + [currentDocumentId, getChangeTracker, updateEntries] + ) + ) + useSocketListener( + socket, + 'new-comment', + useCallback( + (threadId: ThreadId, comment: ReviewPanelCommentThreadMessageApi) => { + setCommentThreads(prevState => { + const { submitting: _, ...thread } = getThread(threadId) + thread.messages = [...thread.messages] + thread.messages.push(formatComment(comment)) + return { ...prevState, [threadId]: thread } + }) + handleLayoutChange({ async: true }) + }, + [getThread] + ) + ) + useSocketListener( + socket, + 'new-comment-threads', + useCallback( + (threads: ReviewPanelCommentThreadsApi) => { + setCommentThreads(prevState => { + const newThreads = { ...prevState } + for (const threadIdString of Object.keys(threads)) { + const threadId = threadIdString as ThreadId + const { submitting: _, ...thread } = getThread(threadId) + // Replace already loaded messages with the server provided ones + thread.messages = threads[threadId].messages.map(formatComment) + newThreads[threadId] = thread + } + return newThreads + }) + handleLayoutChange({ async: true }) + }, + [getThread] + ) + ) + + const openSubView = useRef('cur_file') + useEffect(() => { + if (!reviewPanelOpen) { + // Always show current file when not open, but save current state + setSubView(prevState => { + openSubView.current = prevState + return 'cur_file' + }) + } else { + // Reset back to what we had when previously open + setSubView(openSubView.current) + } + handleLayoutChange({ async: true, animate: false }) + }, [reviewPanelOpen]) + + const canRefreshRanges = useRef(false) + const prevSubView = useRef(subView) + const initializedPrevSubView = useRef(false) + useEffect(() => { + // Prevent setting a computed value for `prevSubView` on mount + if (!initializedPrevSubView.current) { + initializedPrevSubView.current = true + return + } + prevSubView.current = subView === 'cur_file' ? 'overview' : 'cur_file' + // Allow refreshing ranges once for each `subView` change + canRefreshRanges.current = true + }, [subView]) + + useEffect(() => { + if (subView === 'overview' && canRefreshRanges.current) { + canRefreshRanges.current = false + + setIsOverviewLoading(true) + refreshRanges().finally(() => { + setIsOverviewLoading(false) + }) + } + }, [subView, refreshRanges]) + + useEffect(() => { + if (subView === 'cur_file' && prevSubView.current === 'overview') { + dispatchReviewPanelEvent('overview-closed', subView) + } + }, [subView]) + + useEffect(() => { + if (Object.keys(users).length) { + handleLayoutChange({ async: true }) + } + }, [users]) + + const values = useMemo( + () => ({ + collapsed, + commentThreads, + entries, + isAddingComment, + loadingThreads, + nVisibleSelectedChanges, + permissions, + users, + resolvedComments, + shouldCollapse, + navHeight, + toolbarHeight, + subView, + wantTrackChanges, + isOverviewLoading, + openDocId: currentDocumentId, + lineHeight, + trackChangesState, + trackChangesOnForEveryone, + trackChangesOnForGuests, + trackChangesForGuestsAvailable, + formattedProjectMembers, + layoutSuspended, + unsavedComment, + layoutToLeft, + }), + [ + collapsed, + commentThreads, + entries, + isAddingComment, + loadingThreads, + nVisibleSelectedChanges, + permissions, + users, + resolvedComments, + shouldCollapse, + navHeight, + toolbarHeight, + subView, + wantTrackChanges, + isOverviewLoading, + currentDocumentId, + lineHeight, + trackChangesState, + trackChangesOnForEveryone, + trackChangesOnForGuests, + trackChangesForGuestsAvailable, + formattedProjectMembers, + layoutSuspended, + unsavedComment, + layoutToLeft, + ] + ) + + const updaterFns = useMemo( + () => ({ + handleSetSubview, + handleLayoutChange, + gotoEntry, + resolveComment, + submitReply, + acceptChanges, + rejectChanges, + toggleReviewPanel, + bulkAcceptActions, + bulkRejectActions, + saveEdit, + submitNewComment, + deleteComment, + unresolveComment, + refreshResolvedCommentsDropdown: refreshRanges, + deleteThread, + toggleTrackChangesForEveryone, + toggleTrackChangesForUser, + toggleTrackChangesForGuests, + setCollapsed, + setShouldCollapse, + setIsAddingComment, + setNavHeight, + setToolbarHeight, + setLayoutSuspended, + setUnsavedComment, + }), + [ + handleSetSubview, + gotoEntry, + resolveComment, + submitReply, + acceptChanges, + rejectChanges, + toggleReviewPanel, + bulkAcceptActions, + bulkRejectActions, + saveEdit, + submitNewComment, + deleteComment, + unresolveComment, + refreshRanges, + deleteThread, + toggleTrackChangesForEveryone, + toggleTrackChangesForUser, + toggleTrackChangesForGuests, + setCollapsed, + setShouldCollapse, + setIsAddingComment, + setNavHeight, + setToolbarHeight, + setLayoutSuspended, + setUnsavedComment, + ] + ) + + return { values, updaterFns } +} + +export default useReviewPanelState diff --git a/services/web/modules/track-changes/app/src/TrackChangesController.js b/services/web/modules/track-changes/app/src/TrackChangesController.js new file mode 100644 index 00000000000..12cbb57da49 --- /dev/null +++ b/services/web/modules/track-changes/app/src/TrackChangesController.js @@ -0,0 +1,194 @@ +const ChatApiHandler = require('../../../../app/src/Features/Chat/ChatApiHandler') +const ChatManager = require('../../../../app/src/Features/Chat/ChatManager') +const EditorRealTimeController = require('../../../../app/src/Features/Editor/EditorRealTimeController') +const SessionManager = require('../../../../app/src/Features/Authentication/SessionManager') +const UserInfoManager = require('../../../../app/src/Features/User/UserInfoManager') +const DocstoreManager = require('../../../../app/src/Features/Docstore/DocstoreManager') +const DocumentUpdaterHandler = require('../../../../app/src/Features/DocumentUpdater/DocumentUpdaterHandler') +const CollaboratorsGetter = require('../../../../app/src/Features/Collaborators/CollaboratorsGetter') +const { Project } = require('../../../../app/src/models/Project') +const pLimit = require('p-limit') + +function _transformId(doc) { + if (doc._id) { + doc.id = doc._id; + delete doc._id; + } + return doc; +} + +const TrackChangesController = { + async trackChanges(req, res, next) { + try { + const { project_id } = req.params + let state = req.body.on || req.body.on_for + if (req.body.on_for_guests && !req.body.on) state.__guests__ = true + await Project.updateOne({_id: project_id}, {track_changes: state}).exec() //do not wait? + EditorRealTimeController.emitToRoom(project_id, 'toggle-track-changes', state) + res.sendStatus(204) + } catch (err) { + next(err) + } + }, + async acceptChanges(req, res, next) { + try { + const { project_id, doc_id } = req.params + const change_ids = req.body.change_ids + EditorRealTimeController.emitToRoom(project_id, 'accept-changes', doc_id, change_ids) + await DocumentUpdaterHandler.promises.acceptChanges(project_id, doc_id, change_ids) + res.sendStatus(204) + } catch (err) { + next(err) + } + }, + async getAllRanges(req, res, next) { + try { + const { project_id } = req.params + // Flushing the project to mongo is not ideal. Is it possible to fetch the ranges from redis? + await DocumentUpdaterHandler.promises.flushProjectToMongo(project_id) + const ranges = await DocstoreManager.promises.getAllRanges(project_id) + res.json(ranges.map(_transformId)) + } catch (err) { + next(err) + } + }, + async getChangesUsers(req, res, next) { + try { + const { project_id } = req.params + const memberIds = await CollaboratorsGetter.promises.getMemberIds(project_id) + // FIXME: Fails to display names in changes made by former project collaborators. + // See the alternative below. However, it requires flushing the project to mongo, which is not ideal. + const limit = pLimit(3) + const users = await Promise.all( + memberIds.map(memberId => + limit(async () => { + const user = await UserInfoManager.promises.getPersonalInfo(memberId) + return user + }) + ) + ) + users.push({_id: null}) // An anonymous user won't cause any harm + res.json(users.map(_transformId)) + } catch (err) { + next(err) + } + }, +/* + async getChangesUsers(req, res, next) { + try { + const { project_id } = req.params + await DocumentUpdaterHandler.promises.flushProjectToMongo(project_id) + const memberIds = new Set() + const ranges = await DocstoreManager.promises.getAllRanges(project_id) + ranges.forEach(range => { + ;[...range.ranges?.changes || [], ...range.ranges?.comments || []].forEach(item => { + memberIds.add(item.metadata?.user_id) + }) + }) + const limit = pLimit(3) + const users = await Promise.all( + [...memberIds].map(memberId => + limit(async () => { + if( memberId !== "anonymous-user") { + return await UserInfoManager.promises.getPersonalInfo(memberId) + } else { + return {_id: null} + } + }) + ) + ) + res.json(users.map(_transformId)) + } catch (err) { + next(err) + } + }, +*/ + async getThreads(req, res, next) { + try { + const { project_id } = req.params + const messages = await ChatApiHandler.promises.getThreads(project_id) + await ChatManager.promises.injectUserInfoIntoThreads(messages) + res.json(messages) + } catch (err) { + next(err) + } + }, + async sendComment(req, res, next) { + try { + const { project_id, thread_id } = req.params + const { content } = req.body + const user_id = SessionManager.getLoggedInUserId(req.session) + if (!user_id) throw new Error('no logged-in user') + const message = await ChatApiHandler.promises.sendComment(project_id, thread_id, user_id, content) + message.user = await UserInfoManager.promises.getPersonalInfo(user_id) + EditorRealTimeController.emitToRoom(project_id, 'new-comment', thread_id, message) + res.sendStatus(204) + } catch (err) { + next(err); + } + }, + async editMessage(req, res, next) { + try { + const { project_id, thread_id, message_id } = req.params + const { content } = req.body + const user_id = SessionManager.getLoggedInUserId(req.session) + if (!user_id) throw new Error('no logged-in user') + await ChatApiHandler.promises.editMessage(project_id, thread_id, message_id, user_id, content) + EditorRealTimeController.emitToRoom(project_id, 'edit-message', thread_id, message_id, content) + res.sendStatus(204) + } catch (err) { + next(err) + } + }, + async deleteMessage(req, res, next) { + try { + const { project_id, thread_id, message_id } = req.params + await ChatApiHandler.promises.deleteMessage(project_id, thread_id, message_id) + EditorRealTimeController.emitToRoom(project_id, 'delete-message', thread_id, message_id) + res.sendStatus(204) + } catch (err) { + next(err) + } + }, + async resolveThread(req, res, next) { + try { + const { project_id, doc_id, thread_id } = req.params + const user_id = SessionManager.getLoggedInUserId(req.session) + if (!user_id) throw new Error('no logged-in user') + const user = await UserInfoManager.promises.getPersonalInfo(user_id) + await ChatApiHandler.promises.resolveThread(project_id, thread_id, user_id) + EditorRealTimeController.emitToRoom(project_id, 'resolve-thread', thread_id, user) + await DocumentUpdaterHandler.promises.resolveThread(project_id, doc_id, thread_id, user_id) + res.sendStatus(204); + } catch (err) { + next(err); + } + }, + async reopenThread(req, res, next) { + try { + const { project_id, doc_id, thread_id } = req.params + const user_id = SessionManager.getLoggedInUserId(req.session) + if (!user_id) throw new Error('no logged-in user') + await ChatApiHandler.promises.reopenThread(project_id, thread_id) + EditorRealTimeController.emitToRoom(project_id, 'reopen-thread', thread_id) + await DocumentUpdaterHandler.promises.reopenThread(project_id, doc_id, thread_id, user_id) + res.sendStatus(204) + } catch (err) { + next(err) + } + }, + async deleteThread(req, res, next) { + try { + const { project_id, doc_id, thread_id } = req.params + const user_id = SessionManager.getLoggedInUserId(req.session) + if (!user_id) throw new Error('no logged-in user') + await ChatApiHandler.promises.deleteThread(project_id, thread_id) + EditorRealTimeController.emitToRoom(project_id, 'delete-thread', thread_id) + await DocumentUpdaterHandler.promises.deleteThread(project_id, doc_id, thread_id, user_id) + res.sendStatus(204) + } catch (err) { + next(err) + } + }, +} +module.exports = TrackChangesController diff --git a/services/web/modules/track-changes/app/src/TrackChangesRouter.js b/services/web/modules/track-changes/app/src/TrackChangesRouter.js new file mode 100644 index 00000000000..3791e251a1a --- /dev/null +++ b/services/web/modules/track-changes/app/src/TrackChangesRouter.js @@ -0,0 +1,72 @@ +const logger = require('@overleaf/logger') +const AuthorizationMiddleware = require('../../../../app/src/Features/Authorization/AuthorizationMiddleware') +const TrackChangesController = require('./TrackChangesController') + +module.exports = { + apply(webRouter) { + logger.debug({}, 'Init track-changes router') + + webRouter.post('/project/:project_id/track_changes', + AuthorizationMiddleware.blockRestrictedUserFromProject, + AuthorizationMiddleware.ensureUserCanReadProject, + TrackChangesController.trackChanges + ) + webRouter.post('/project/:project_id/doc/:doc_id/changes/accept', + AuthorizationMiddleware.blockRestrictedUserFromProject, + AuthorizationMiddleware.ensureUserCanReadProject, + TrackChangesController.acceptChanges + ) + webRouter.get('/project/:project_id/ranges', + AuthorizationMiddleware.blockRestrictedUserFromProject, + AuthorizationMiddleware.ensureUserCanReadProject, + TrackChangesController.getAllRanges + ) + webRouter.get('/project/:project_id/changes/users', + AuthorizationMiddleware.blockRestrictedUserFromProject, + AuthorizationMiddleware.ensureUserCanReadProject, + TrackChangesController.getChangesUsers + ) + webRouter.get( + '/project/:project_id/threads', + AuthorizationMiddleware.blockRestrictedUserFromProject, + AuthorizationMiddleware.ensureUserCanReadProject, + TrackChangesController.getThreads + ) + webRouter.post( + '/project/:project_id/thread/:thread_id/messages', + AuthorizationMiddleware.blockRestrictedUserFromProject, + AuthorizationMiddleware.ensureUserCanReadProject, + TrackChangesController.sendComment + ) + webRouter.post( + '/project/:project_id/thread/:thread_id/messages/:message_id/edit', + AuthorizationMiddleware.blockRestrictedUserFromProject, + AuthorizationMiddleware.ensureUserCanReadProject, + TrackChangesController.editMessage + ) + webRouter.delete( + '/project/:project_id/thread/:thread_id/messages/:message_id', + AuthorizationMiddleware.blockRestrictedUserFromProject, + AuthorizationMiddleware.ensureUserCanReadProject, + TrackChangesController.deleteMessage + ) + webRouter.post( + '/project/:project_id/doc/:doc_id/thread/:thread_id/resolve', + AuthorizationMiddleware.blockRestrictedUserFromProject, + AuthorizationMiddleware.ensureUserCanReadProject, + TrackChangesController.resolveThread + ) + webRouter.post( + '/project/:project_id/doc/:doc_id/thread/:thread_id/reopen', + AuthorizationMiddleware.blockRestrictedUserFromProject, + AuthorizationMiddleware.ensureUserCanReadProject, + TrackChangesController.reopenThread + ) + webRouter.delete( + '/project/:project_id/doc/:doc_id/thread/:thread_id', + AuthorizationMiddleware.blockRestrictedUserFromProject, + AuthorizationMiddleware.ensureUserCanReadProject, + TrackChangesController.deleteThread + ) + }, +} diff --git a/services/web/modules/track-changes/index.js b/services/web/modules/track-changes/index.js new file mode 100644 index 00000000000..aa9e6a73dad --- /dev/null +++ b/services/web/modules/track-changes/index.js @@ -0,0 +1,2 @@ +const TrackChangesRouter = require('./app/src/TrackChangesRouter') +module.exports = { router : TrackChangesRouter } From b68d12987dc9dcac10eb2b4eaa34283b55b78553 Mon Sep 17 00:00:00 2001 From: yu-i-i Date: Wed, 28 Jan 2026 02:43:00 +0100 Subject: [PATCH 003/106] Enable autocomplete of reference keys feature --- develop/README.md | 1 + develop/dev.env | 1 + develop/docker-compose.dev.yml | 11 + develop/docker-compose.yml | 8 + server-ce/config/env.sh | 1 + server-ce/runit/references-overleaf/run | 12 + server-ce/services.js | 3 + services/references/.nvmrc | 1 + services/references/Dockerfile | 31 + services/references/LICENSE | 662 ++++++ services/references/Makefile | 156 ++ services/references/README.md | 10 + services/references/app.js | 36 + .../app/js/ReferencesAPIController.js | 42 + services/references/app/js/bib2json.js | 1967 +++++++++++++++++ services/references/buildscript.txt | 6 + .../references/config/settings.defaults.cjs | 9 + services/references/docker-compose.ci.yml | 38 + services/references/docker-compose.yml | 40 + services/references/package.json | 20 + services/references/tsconfig.json | 12 + services/web/config/settings.defaults.js | 3 + 22 files changed, 3070 insertions(+) create mode 100755 server-ce/runit/references-overleaf/run create mode 100644 services/references/.nvmrc create mode 100644 services/references/Dockerfile create mode 100644 services/references/LICENSE create mode 100644 services/references/Makefile create mode 100644 services/references/README.md create mode 100644 services/references/app.js create mode 100644 services/references/app/js/ReferencesAPIController.js create mode 100644 services/references/app/js/bib2json.js create mode 100644 services/references/buildscript.txt create mode 100644 services/references/config/settings.defaults.cjs create mode 100644 services/references/docker-compose.ci.yml create mode 100644 services/references/docker-compose.yml create mode 100644 services/references/package.json create mode 100644 services/references/tsconfig.json diff --git a/develop/README.md b/develop/README.md index b6e0769aec0..e1701a3b6e8 100644 --- a/develop/README.md +++ b/develop/README.md @@ -65,6 +65,7 @@ each service: | `filestore` | 9235 | | `notifications` | 9236 | | `real-time` | 9237 | +| `references` | 9238 | | `history-v1` | 9239 | | `project-history` | 9240 | diff --git a/develop/dev.env b/develop/dev.env index f2e6004eece..f8ec3d67475 100644 --- a/develop/dev.env +++ b/develop/dev.env @@ -17,6 +17,7 @@ QUEUES_REDIS_HOST=redis DSMP_REDIS_HOST=redis REALTIME_HOST=real-time REDIS_HOST=redis +REFERENCES_HOST=references SESSION_SECRET=foo V1_HISTORY_HOST=history-v1 WEBPACK_HOST=webpack diff --git a/develop/docker-compose.dev.yml b/develop/docker-compose.dev.yml index 5315cddcc3c..c72d91d0731 100644 --- a/develop/docker-compose.dev.yml +++ b/develop/docker-compose.dev.yml @@ -112,6 +112,17 @@ services: - ../services/real-time/app.js:/overleaf/services/real-time/app.js - ../services/real-time/config:/overleaf/services/real-time/config + references: + command: ["node", "--watch", "app.js"] + environment: + - NODE_OPTIONS=--inspect=0.0.0.0:9229 + ports: + - "127.0.0.1:9238:9229" + volumes: + - ../services/references/app:/overleaf/services/references/app + - ../services/references/config:/overleaf/services/references/config + - ../services/references/app.js:/overleaf/services/references/app.js + web: command: ["node", "--watch", "app.mjs", "--watch-locales"] environment: diff --git a/develop/docker-compose.yml b/develop/docker-compose.yml index df63fca6a25..3c8c0787192 100644 --- a/develop/docker-compose.yml +++ b/develop/docker-compose.yml @@ -139,6 +139,13 @@ services: volumes: - redis-data:/data + references: + build: + context: .. + dockerfile: services/references/Dockerfile + env_file: + - dev.env + web: build: context: .. @@ -169,6 +176,7 @@ services: - notifications - project-history - real-time + - references webpack: build: diff --git a/server-ce/config/env.sh b/server-ce/config/env.sh index b12ca242d34..81cebe4caae 100644 --- a/server-ce/config/env.sh +++ b/server-ce/config/env.sh @@ -9,5 +9,6 @@ export HISTORY_V1_HOST=127.0.0.1 export NOTIFICATIONS_HOST=127.0.0.1 export PROJECT_HISTORY_HOST=127.0.0.1 export REALTIME_HOST=127.0.0.1 +export REFERENCES_HOST=127.0.0.1 export WEB_HOST=127.0.0.1 export WEB_API_HOST=127.0.0.1 diff --git a/server-ce/runit/references-overleaf/run b/server-ce/runit/references-overleaf/run new file mode 100755 index 00000000000..875023df9fa --- /dev/null +++ b/server-ce/runit/references-overleaf/run @@ -0,0 +1,12 @@ +#!/bin/bash + +NODE_PARAMS="" +if [ "$DEBUG_NODE" == "true" ]; then + echo "running debug - references" + NODE_PARAMS="--inspect=0.0.0.0:30560" +fi + +source /etc/overleaf/env.sh +export LISTEN_ADDRESS=127.0.0.1 + +exec /sbin/setuser www-data /usr/bin/node $NODE_PARAMS /overleaf/services/references/app.js >> /var/log/overleaf/references.log 2>&1 diff --git a/server-ce/services.js b/server-ce/services.js index d0b0a9c0765..e0282f3bad2 100644 --- a/server-ce/services.js +++ b/server-ce/services.js @@ -29,6 +29,9 @@ module.exports = [ { name: 'project-history', }, + { + name: 'references', + }, { name: 'history-v1', }, diff --git a/services/references/.nvmrc b/services/references/.nvmrc new file mode 100644 index 00000000000..91d5f6ff8e3 --- /dev/null +++ b/services/references/.nvmrc @@ -0,0 +1 @@ +22.18.0 diff --git a/services/references/Dockerfile b/services/references/Dockerfile new file mode 100644 index 00000000000..cfff02e7e21 --- /dev/null +++ b/services/references/Dockerfile @@ -0,0 +1,31 @@ +# This file was auto-generated, do not edit it directly. +# Instead run bin/update_build_scripts from +# https://github.com/overleaf/internal/ + +FROM node:22.18.0 AS base + +WORKDIR /overleaf/services/references + +# Google Cloud Storage needs a writable $HOME/.config for resumable uploads +# (see https://googleapis.dev/nodejs/storage/latest/File.html#createWriteStream) +RUN mkdir /home/node/.config && chown node:node /home/node/.config + +FROM base AS app + +COPY package.json package-lock.json /overleaf/ +COPY libraries/logger/package.json /overleaf/libraries/logger/package.json +COPY libraries/metrics/package.json /overleaf/libraries/metrics/package.json +COPY libraries/settings/package.json /overleaf/libraries/settings/package.json +COPY services/references/package.json /overleaf/services/references/package.json +COPY patches/ /overleaf/patches/ + +RUN cd /overleaf && npm ci --quiet +COPY libraries/logger/ /overleaf/libraries/logger/ +COPY libraries/metrics/ /overleaf/libraries/metrics/ +COPY libraries/settings/ /overleaf/libraries/settings/ +COPY services/references/ /overleaf/services/references/ + +FROM app +USER node + +CMD ["node", "--expose-gc", "app.js"] diff --git a/services/references/LICENSE b/services/references/LICENSE new file mode 100644 index 00000000000..ac8619dcb91 --- /dev/null +++ b/services/references/LICENSE @@ -0,0 +1,662 @@ + + GNU AFFERO GENERAL PUBLIC LICENSE + Version 3, 19 November 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU Affero General Public License is a free, copyleft license for +software and other kinds of works, specifically designed to ensure +cooperation with the community in the case of network server software. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +our General Public Licenses are intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + Developers that use our General Public Licenses protect your rights +with two steps: (1) assert copyright on the software, and (2) offer +you this License which gives you legal permission to copy, distribute +and/or modify the software. + + A secondary benefit of defending all users' freedom is that +improvements made in alternate versions of the program, if they +receive widespread use, become available for other developers to +incorporate. Many developers of free software are heartened and +encouraged by the resulting cooperation. However, in the case of +software used on network servers, this result may fail to come about. +The GNU General Public License permits making a modified version and +letting the public access it on a server without ever releasing its +source code to the public. + + The GNU Affero General Public License is designed specifically to +ensure that, in such cases, the modified source code becomes available +to the community. It requires the operator of a network server to +provide the source code of the modified version running there to the +users of that server. Therefore, public use of a modified version, on +a publicly accessible server, gives the public access to the source +code of the modified version. + + An older license, called the Affero General Public License and +published by Affero, was designed to accomplish similar goals. This is +a different license, not a version of the Affero GPL, but Affero has +released a new version of the Affero GPL which permits relicensing under +this license. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU Affero General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Remote Network Interaction; Use with the GNU General Public License. + + Notwithstanding any other provision of this License, if you modify the +Program, your modified version must prominently offer all users +interacting with it remotely through a computer network (if your version +supports such interaction) an opportunity to receive the Corresponding +Source of your version by providing access to the Corresponding Source +from a network server at no charge, through some standard or customary +means of facilitating copying of software. This Corresponding Source +shall include the Corresponding Source for any work covered by version 3 +of the GNU General Public License that is incorporated pursuant to the +following paragraph. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the work with which it is combined will remain governed by version +3 of the GNU General Public License. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU Affero General Public License from time to time. Such new versions +will be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU Affero General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU Affero General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU Affero General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If your software can interact with users remotely through a computer +network, you should also make sure that it provides a way for users to +get its source. For example, if your program is a web application, its +interface could display a "Source" link that leads users to an archive +of the code. There are many ways you could offer source, and different +solutions will be better for different programs; see section 13 for the +specific requirements. + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU AGPL, see +. diff --git a/services/references/Makefile b/services/references/Makefile new file mode 100644 index 00000000000..d0e4f27efd0 --- /dev/null +++ b/services/references/Makefile @@ -0,0 +1,156 @@ +# This file was auto-generated, do not edit it directly. +# Instead run bin/update_build_scripts from +# https://github.com/overleaf/internal/ + +BUILD_NUMBER ?= local +BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD) +PROJECT_NAME = references +BUILD_DIR_NAME = $(shell pwd | xargs basename | tr -cd '[a-zA-Z0-9_.\-]') + +DOCKER_COMPOSE_FLAGS ?= -f docker-compose.yml +DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \ + BRANCH_NAME=$(BRANCH_NAME) \ + PROJECT_NAME=$(PROJECT_NAME) \ + MOCHA_GREP=${MOCHA_GREP} \ + docker compose ${DOCKER_COMPOSE_FLAGS} + +COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE ?= test_acceptance_$(BUILD_DIR_NAME) +DOCKER_COMPOSE_TEST_ACCEPTANCE = \ + COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_ACCEPTANCE) $(DOCKER_COMPOSE) + +COMPOSE_PROJECT_NAME_TEST_UNIT ?= test_unit_$(BUILD_DIR_NAME) +DOCKER_COMPOSE_TEST_UNIT = \ + COMPOSE_PROJECT_NAME=$(COMPOSE_PROJECT_NAME_TEST_UNIT) $(DOCKER_COMPOSE) + +clean: + -docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) + -docker rmi us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) + -$(DOCKER_COMPOSE_TEST_UNIT) down --rmi local + -$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down --rmi local + +HERE=$(shell pwd) +MONOREPO=$(shell cd ../../ && pwd) +# Run the linting commands in the scope of the monorepo. +# Eslint and prettier (plus some configs) are on the root. +RUN_LINTING = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(HERE) node:22.18.0 npm run --silent + +RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) npm run --silent + +# Same but from the top of the monorepo +RUN_LINTING_MONOREPO = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(MONOREPO) node:22.18.0 npm run --silent + +SHELLCHECK_OPTS = \ + --shell=bash \ + --external-sources +SHELLCHECK_COLOR := $(if $(CI),--color=never,--color) +SHELLCHECK_FILES := { git ls-files "*.sh" -z; git grep -Plz "\A\#\!.*bash"; } | sort -zu + +shellcheck: + @$(SHELLCHECK_FILES) | xargs -0 -r docker run --rm -v $(HERE):/mnt -w /mnt \ + koalaman/shellcheck:stable $(SHELLCHECK_OPTS) $(SHELLCHECK_COLOR) + +shellcheck_fix: + @$(SHELLCHECK_FILES) | while IFS= read -r -d '' file; do \ + diff=$$(docker run --rm -v $(HERE):/mnt -w /mnt koalaman/shellcheck:stable $(SHELLCHECK_OPTS) --format=diff "$$file" 2>/dev/null); \ + if [ -n "$$diff" ] && ! echo "$$diff" | patch -p1 >/dev/null 2>&1; then echo "\033[31m$$file\033[0m"; \ + elif [ -n "$$diff" ]; then echo "$$file"; \ + else echo "\033[2m$$file\033[0m"; fi \ + done + +format: + $(RUN_LINTING) format + +format_ci: + $(RUN_LINTING_CI) format + +format_fix: + $(RUN_LINTING) format:fix + +lint: + $(RUN_LINTING) lint + +lint_ci: + $(RUN_LINTING_CI) lint + +lint_fix: + $(RUN_LINTING) lint:fix + +typecheck: + $(RUN_LINTING) types:check + +typecheck_ci: + $(RUN_LINTING_CI) types:check + +test: format lint typecheck shellcheck test_unit test_acceptance + +test_unit: +ifneq (,$(wildcard test/unit)) + $(DOCKER_COMPOSE_TEST_UNIT) run --rm test_unit + $(MAKE) test_unit_clean +endif + +test_clean: test_unit_clean +test_unit_clean: +ifneq (,$(wildcard test/unit)) + $(DOCKER_COMPOSE_TEST_UNIT) down -v -t 0 +endif + +test_acceptance: test_acceptance_clean test_acceptance_pre_run test_acceptance_run + $(MAKE) test_acceptance_clean + +test_acceptance_debug: test_acceptance_clean test_acceptance_pre_run test_acceptance_run_debug + $(MAKE) test_acceptance_clean + +test_acceptance_run: +ifneq (,$(wildcard test/acceptance)) + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance +endif + +test_acceptance_run_debug: +ifneq (,$(wildcard test/acceptance)) + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run -p 127.0.0.9:19999:19999 --rm test_acceptance npm run test:acceptance -- --inspect=0.0.0.0:19999 --inspect-brk +endif + +test_clean: test_acceptance_clean +test_acceptance_clean: + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) down -v -t 0 + +test_acceptance_pre_run: +ifneq (,$(wildcard test/acceptance/js/scripts/pre-run)) + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance test/acceptance/js/scripts/pre-run +endif + +benchmarks: + $(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance npm run benchmarks + +build: + docker build \ + --pull \ + --build-arg BUILDKIT_INLINE_CACHE=1 \ + --tag ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \ + --tag us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \ + --tag us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME) \ + --cache-from us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME) \ + --cache-from us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):main \ + --file Dockerfile \ + ../.. + +tar: + $(DOCKER_COMPOSE) up tar + +publish: + + docker push $(DOCKER_REPO)/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) + + +.PHONY: clean \ + format format_fix \ + lint lint_fix \ + build_types typecheck \ + lint_ci format_ci typecheck_ci \ + shellcheck shellcheck_fix \ + test test_clean test_unit test_unit_clean \ + test_acceptance test_acceptance_debug test_acceptance_pre_run \ + test_acceptance_run test_acceptance_run_debug test_acceptance_clean \ + benchmarks \ + build tar publish \ diff --git a/services/references/README.md b/services/references/README.md new file mode 100644 index 00000000000..41844d259ae --- /dev/null +++ b/services/references/README.md @@ -0,0 +1,10 @@ +overleaf/references +=============== + +An API for providing citation-keys from user bib-files + +License +======= +The code in this repository is released under the GNU AFFERO GENERAL PUBLIC LICENSE, version 3. + +Based on https://github.com/overleaf/overleaf/commit/9964aebc794f9fd7ce1373ab3484f6b33b061af3 diff --git a/services/references/app.js b/services/references/app.js new file mode 100644 index 00000000000..d2b392afbbc --- /dev/null +++ b/services/references/app.js @@ -0,0 +1,36 @@ +import '@overleaf/metrics/initialize.js' + +import express from 'express' +import Settings from '@overleaf/settings' +import logger from '@overleaf/logger' +import metrics from '@overleaf/metrics' +import ReferencesAPIController from './app/js/ReferencesAPIController.js' +import bodyParser from 'body-parser' + +const app = express() +metrics.injectMetricsRoute(app) + +app.use(bodyParser.json({ limit: '2mb' })) +app.use(metrics.http.monitor(logger)) + +app.post('/project/:project_id/index', ReferencesAPIController.index) +app.get('/status', (req, res) => res.send({ status: 'references api is up' })) + +const host = Settings.internal.references.host +const port = Settings.internal.references.port + +logger.debug('Listening at', { host, port }) + +const server = app.listen(port, host, function (error) { + if (error) { + throw error + } + logger.info({ host, port }, 'references HTTP server starting up') +}) + +process.on('SIGTERM', () => { + server.close(() => { + logger.info({ host, port }, 'references HTTP server closed') + metrics.close() + }) +}) diff --git a/services/references/app/js/ReferencesAPIController.js b/services/references/app/js/ReferencesAPIController.js new file mode 100644 index 00000000000..ac51ca6bbdf --- /dev/null +++ b/services/references/app/js/ReferencesAPIController.js @@ -0,0 +1,42 @@ +import logger from '@overleaf/logger' +import BibtexParser from './bib2json.js' + +export default { + async index(req, res) { + const { docUrls, fullIndex } = req.body + try { + const responses = await Promise.all( + docUrls.map(async (docUrl) => { + try { + const response = await fetch(docUrl) + if (!response.ok) { + throw new Error(`HTTP error! status: ${response.status}`) + } + return response.text() + } catch (error) { + logger.error({ error }, "Failed to fetch document from URL: " + docUrl) + return null + } + }) + ) + const keys = [] + for (const body of responses) { + if (!body) continue + + try { + const parsedEntries = BibtexParser(body).entries + const ks = parsedEntries + .filter(entry => entry.EntryKey) + .map(entry => entry.EntryKey) + keys.push(...ks) + } catch (error) { + logger.error({ error }, "bib file skipped.") + } + } + res.status(200).json({ keys }) + } catch (error) { + logger.error({ error }, "Unexpected error during indexing process.") + res.status(500).json({ error: "Failed to process bib files." }) + } + } +} diff --git a/services/references/app/js/bib2json.js b/services/references/app/js/bib2json.js new file mode 100644 index 00000000000..99cfcf70ee2 --- /dev/null +++ b/services/references/app/js/bib2json.js @@ -0,0 +1,1967 @@ +/* eslint-disable */ +/** + * Parser.js + * Copyright 2012-13 Mayank Lahiri + * mlahiri@gmail.com + * Released under the BSD License. + * + * Modifications 2016 Sharelatex + * Modifications 2017-2020 Overleaf + * + * A forgiving Bibtex parser that can: + * + * (1) operate in streaming or block mode, extracting entries as dictionaries. + * (2) convert Latex special characters to UTF-8. + * (3) best-effort parse malformed entries. + * (4) run in a CommonJS environment or a browser, without any dependencies. + * (5) be advanced-compiled by Google Closure Compiler. + * + * Handwritten as a labor of love, not auto-generated from a grammar. + * + * Modes of usage: + * + * (1) Synchronous, string + * + * var entries = BibtexParser(text); + * console.log(entries); + * + * (2) Asynchronous, stream + * + * function entryCallback(entry) { console.log(entry); } + * var parser = new BibtexParser(entryCallback); + * parser.parse(chunk1); + * parser.parse(chunk2); + * ... + * + * @param {text|function(Object)} arg0 Either a Bibtex string or callback + * function for processing parsed entries. + * @param {array} allowedKeys optimization: do not output key/value pairs that are not on this allowlist + * @constructor + */ +function BibtexParser(arg0, allowedKeys) { + // Determine how this function is to be used + if (typeof arg0 === 'string') { + // Passed a string, synchronous call without 'new' + const entries = [] + function accumulator(entry) { + entries.push(entry) + } + const parser = new BibtexParser(accumulator, allowedKeys) + parser.parse(arg0) + return { + entries, + errors: parser.getErrors(), + } + } + if (typeof arg0 !== 'function') { + throw 'Invalid parser construction.' + } + this.ALLOWEDKEYS_ = allowedKeys || [] + this.reset_(arg0) + this.initMacros_() + return this +} + +/** @enum {number} */ +BibtexParser.prototype.STATES_ = { + ENTRY_OR_JUNK: 0, + OBJECT_TYPE: 1, + ENTRY_KEY: 2, + KV_KEY: 3, + EQUALS: 4, + KV_VALUE: 5, +} +BibtexParser.prototype.reset_ = function (arg0) { + /** @private */ this.DATA_ = {} + /** @private */ this.CALLBACK_ = arg0 + /** @private */ this.CHAR_ = 0 + /** @private */ this.LINE_ = 1 + /** @private */ this.CHAR_IN_LINE_ = 0 + /** @private */ this.SKIPWS_ = true + /** @private */ this.SKIPCOMMENT_ = true + /** @private */ this.SKIPKVPAIR_ = false + /** @private */ this.PARSETMP_ = {} + /** @private */ this.SKIPTILLEOL_ = false + /** @private */ this.VALBRACES_ = null + /** @private */ this.BRACETYPE_ = null + /** @private */ this.BRACECOUNT_ = 0 + /** @private */ this.STATE_ = this.STATES_.ENTRY_OR_JUNK + /** @private */ this.ERRORS_ = [] +} +/** @private */ BibtexParser.prototype.ENTRY_TYPES_ = { + inproceedings: 1, + proceedings: 2, + article: 3, + techreport: 4, + misc: 5, + mastersthesis: 6, + book: 7, + phdthesis: 8, + incollection: 9, + unpublished: 10, + inbook: 11, + manual: 12, + periodical: 13, + booklet: 14, + masterthesis: 15, + conference: 16, + /* additional fields from biblatex */ + artwork: 17, + audio: 18, + bibnote: 19, + bookinbook: 20, + collection: 21, + commentary: 22, + customa: 23, + customb: 24, + customc: 25, + customd: 26, + custome: 27, + customf: 28, + image: 29, + inreference: 30, + jurisdiction: 31, + legal: 32, + legislation: 33, + letter: 34, + movie: 35, + music: 36, + mvbook: 37, + mvcollection: 38, + mvproceedings: 39, + mvreference: 40, + online: 41, + patent: 42, + performance: 43, + reference: 44, + report: 45, + review: 46, + set: 47, + software: 48, + standard: 49, + suppbook: 50, + suppcollection: 51, + thesis: 52, + video: 53, +} +BibtexParser.prototype.initMacros_ = function () { + // macros can be extended by the user via + // @string { macroName = "macroValue" } + /** @private */ this.MACROS_ = { + jan: 'January', + feb: 'February', + mar: 'March', + apr: 'April', + may: 'May', + jun: 'June', + jul: 'July', + aug: 'August', + sep: 'September', + oct: 'October', + nov: 'November', + dec: 'December', + Jan: 'January', + Feb: 'February', + Mar: 'March', + Apr: 'April', + May: 'May', + Jun: 'June', + Jul: 'July', + Aug: 'August', + Sep: 'September', + Oct: 'October', + Nov: 'November', + Dec: 'December', + } +} + +/** + * Gets an array of all errors encountered during parsing. + * Array entries are of the format: + * [ line number, character in line, character in stream, error text ] + * + * @returns Array + * @public + */ +BibtexParser.prototype.getErrors = function () { + return this.ERRORS_ +} + +/** + * Processes a chunk of data + * @public + */ +BibtexParser.prototype.parse = function (chunk) { + for (let i = 0; i < chunk.length; i++) this.processCharacter_(chunk[i]) +} + +/** + * Logs error at current stream position. + * + * @private + */ +BibtexParser.prototype.error_ = function (text) { + this.ERRORS_.push([this.LINE_, this.CHAR_IN_LINE_, this.CHAR_, text]) +} + +/** + * Called after an entire entry has been parsed from the stream. + * Performs post-processing and invokes the entry callback pointed to by + * this.CALLBACK_. Parsed (but unprocessed) entry data is in this.DATA_. + */ +BibtexParser.prototype.processEntry_ = function () { + const data = this.DATA_ + if (data.Fields) + for (const f in data.Fields) { + let raw = data.Fields[f] + + // Convert Latex/Bibtex special characters to UTF-8 equivalents + for (let i = 0; i < this.CHARCONV_.length; i++) { + const re = this.CHARCONV_[i][0] + const rep = this.CHARCONV_[i][1] + raw = raw.replace(re, rep) + } + + // Basic substitutions + raw = raw + .replace(/[\n\r\t]/g, ' ') + .replace(/\s\s+/g, ' ') + .replace(/^\s+|\s+$/g, '') + + // Remove braces and backslashes + const len = raw.length + let processedArr = [] + for (let i = 0; i < len; i++) { + let c = raw[i] + let skip = false + if (c == '\\' && i < len - 1) c = raw[++i] + else { + if (c == '{' || c == '}') skip = true + } + if (!skip) processedArr.push(c) + } + data.Fields[f] = processedArr.join('') + processedArr = null + } + + if (data.ObjectType == 'string') { + for (const f in data.Fields) { + this.MACROS_[f] = data.Fields[f] + } + } else { + // Parsed a new Bibtex entry + this.CALLBACK_(data) + } +} + +/** + * Processes next character in the stream, invoking the callback after + * each entry has been found and processed. + * + * @private + * @param {string} c Next character in input stream + */ +BibtexParser.prototype.processCharacter_ = function (c) { + // Housekeeping + this.CHAR_++ + this.CHAR_IN_LINE_++ + if (c == '\n') { + this.LINE_++ + this.CHAR_IN_LINE_ = 1 + } + + // Convenience states for skipping whitespace when needed + if (this.SKIPTILLEOL_) { + if (c == '\n') this.SKIPTILLEOL_ = false + return + } + if (this.SKIPCOMMENT_ && c == '%') { + this.SKIPTILLEOL_ = true + return + } + if (this.SKIPWS_ && /\s/.test(c)) return + this.SKIPWS_ = false + this.SKIPCOMMENT_ = false + this.SKIPTILLEOL_ = false + + // Main state machine + let AnotherIteration = true + while (AnotherIteration) { + // console.log(this.LINE_, this.CHAR_IN_LINE_, this.STATE_, c) + AnotherIteration = false + switch (this.STATE_) { + // -- Scan for an object marker ('@') + // -- Reset temporary data structure in case previous entry was garbled + case this.STATES_.ENTRY_OR_JUNK: + if (c == '@') { + // SUCCESS: Parsed a valid start-of-object marker. + // NEXT_STATE: OBJECT_TYPE + this.STATE_ = this.STATES_.OBJECT_TYPE + this.DATA_ = { + ObjectType: '', + } + } + this.BRACETYPE_ = null + this.SKIPWS_ = true + this.SKIPCOMMENT_ = true + break + + // Start at first non-whitespace character after start-of-object '@' + // -- Accept [A-Za-z], break on non-matching character + // -- Populate this.DATA_.EntryType and this.DATA_.ObjectType + case this.STATES_.OBJECT_TYPE: + if (/[A-Za-z]/.test(c)) { + this.DATA_.ObjectType += c.toLowerCase() + this.SKIPWS_ = true + this.SKIPCOMMENT_ = true + } else { + // Break from state and validate object type + const ot = this.DATA_.ObjectType + if (ot == 'comment') { + this.STATE_ = this.STATES_.ENTRY_OR_JUNK + } else { + if (ot == 'string') { + this.DATA_.ObjectType = ot + this.DATA_.Fields = {} + this.BRACETYPE_ = c + this.BRACECOUNT_ = 1 + this.STATE_ = this.STATES_.KV_KEY + this.SKIPWS_ = true + this.SKIPCOMMENT_ = true + this.PARSETMP_ = { + Key: '', + } + } else { + if (ot == 'preamble') { + this.STATE_ = this.STATES_.ENTRY_OR_JUNK + } else { + if (ot in this.ENTRY_TYPES_) { + // SUCCESS: Parsed a valid object type. + // NEXT_STATE: ENTRY_KEY + this.DATA_.ObjectType = 'entry' + this.DATA_.EntryType = ot + this.DATA_.EntryKey = '' + this.STATE_ = this.STATES_.ENTRY_KEY + AnotherIteration = true + } else { + // ERROR: Unrecognized object type. + // NEXT_STATE: ENTRY_OR_JUNK + this.error_( + 'Unrecognized object type: "' + this.DATA_.ObjectType + '"' + ) + this.STATE_ = this.STATES_.ENTRY_OR_JUNK + } + } + } + } + } + break + + // Start at first non-alphabetic character after an entry type + // -- Populate this.DATA_.EntryKey + case this.STATES_.ENTRY_KEY: + if ((c === '{' || c === '(') && this.BRACETYPE_ == null) { + this.BRACETYPE_ = c + this.BRACECOUNT_ = 1 + this.SKIPWS_ = true + this.SKIPCOMMENT_ = true + break + } + if (/[,%\s]/.test(c)) { + if (this.DATA_.EntryKey.length < 1) { + // Skip comments and whitespace before entry key + this.SKIPWS_ = true + this.SKIPCOMMENT_ = true + } else { + if (this.BRACETYPE_ == null) { + // ERROR: No opening brace for object + // NEXT_STATE: ENTRY_OR_JUNK + this.error_('No opening brace for object.') + this.STATE_ = this.STATES_.ENTRY_OR_JUNK + } else { + // SUCCESS: Parsed an entry key + // NEXT_STATE: KV_KEY + this.SKIPWS_ = true + this.SKIPCOMMENT_ = true + AnotherIteration = true + this.STATE_ = this.STATES_.KV_KEY + this.PARSETMP_.Key = '' + this.DATA_.Fields = {} + } + } + } else { + this.DATA_.EntryKey += c + this.SKIPWS_ = false + this.SKIPCOMMENT_ = false + } + break + + // Start at first non-whitespace/comment character after entry key. + // -- Populate this.PARSETMP_.Key + case this.STATES_.KV_KEY: + // Test for end of entry + if ( + (c == '}' && this.BRACETYPE_ == '{') || + (c == ')' && this.BRACETYPE_ == '(') + ) { + // SUCCESS: Parsed an entry, possible incomplete + // NEXT_STATE: ENTRY_OR_JUNK + this.processEntry_() + this.SKIPWS_ = true + this.SKIPCOMMENT_ = true + this.STATE_ = this.STATES_.ENTRY_OR_JUNK + break + } + if (/[\-A-Za-z:]/.test(c)) { + // Add to key + this.PARSETMP_.Key += c + this.SKIPWS_ = false + this.SKIPCOMMENT_ = false + } else { + // Either end of key or we haven't encountered start of key + if (this.PARSETMP_.Key.length < 1) { + // Keep going till we see a key + this.SKIPWS_ = true + this.SKIPCOMMENT_ = true + } else { + // SUCCESS: Found full key in K/V pair + // NEXT_STATE: EQUALS + this.SKIPWS_ = true + this.SKIPCOMMENT_ = true + this.STATE_ = this.STATES_.EQUALS + AnotherIteration = true + + if (this.DATA_.ObjectType !== 'string') { + // this entry is not a macro + // normalize the key to lower case + this.PARSETMP_.Key = this.PARSETMP_.Key.toLowerCase() + + // optimization: skip key/value pairs that are not on the allowlist + this.SKIPKVPAIR_ = + // has allowedKeys set + this.ALLOWEDKEYS_.length && + // key is not on the allowlist + this.ALLOWEDKEYS_.indexOf(this.PARSETMP_.Key) === -1 + } else { + this.SKIPKVPAIR_ = false + } + } + } + break + + // Start at first non-alphabetic character after K/V pair key. + case this.STATES_.EQUALS: + if ( + (c == '}' && this.BRACETYPE_ == '{') || + (c == ')' && this.BRACETYPE_ == '(') + ) { + // ERROR: K/V pair with key but no value + // NEXT_STATE: ENTRY_OR_JUNK + this.error_( + 'Key-value pair has key "' + this.PARSETMP_.Key + '", but no value.' + ) + this.processEntry_() + this.SKIPWS_ = true + this.SKIPCOMMENT_ = true + this.STATE_ = this.STATES_.ENTRY_OR_JUNK + break + } + if (c == '=') { + // SUCCESS: found an equal signs separating key and value + // NEXT_STATE: KV_VALUE + this.SKIPWS_ = true + this.SKIPCOMMENT_ = true + this.STATE_ = this.STATES_.KV_VALUE + this.PARSETMP_.Value = [] + this.VALBRACES_ = { '"': [], '{': [] } + } + break + + // Start at first non-whitespace/comment character after '=' + // -- Populate this.PARSETMP_.Value + case this.STATES_.KV_VALUE: + const delim = this.VALBRACES_ + // valueCharsArray is the list of characters that make up the + // current value + const valueCharsArray = this.PARSETMP_.Value + let doneParsingValue = false + + // Test for special characters + if (c == '"' || c == '{' || c == '}' || c == ',') { + if (c == ',') { + // This comma can mean: + // (1) just another comma literal + // (2) end of a macro reference + if (delim['"'].length + delim['{'].length === 0) { + // end of a macro reference + const macro = this.PARSETMP_.Value.join('').trim() + if (macro in this.MACROS_) { + // Successful macro reference + this.PARSETMP_.Value = [this.MACROS_[macro]] + } else { + // Reference to an undefined macro + this.error_('Reference to an undefined macro: ' + macro) + } + doneParsingValue = true + } + } + if (c == '"') { + // This quote can mean: + // (1) opening delimiter + // (2) closing delimiter + // (3) literal, if we have a '{' on the stack + if (delim['"'].length + delim['{'].length === 0) { + // opening delimiter + delim['"'].push(this.CHAR_) + this.SKIPWS_ = false + this.SKIPCOMMENT_ = false + break + } + if ( + delim['"'].length == 1 && + delim['{'].length == 0 && + (valueCharsArray.length == 0 || + valueCharsArray[valueCharsArray.length - 1] != '\\') + ) { + // closing delimiter + doneParsingValue = true + } else { + // literal, add to value + } + } + if (c == '{') { + // This brace can mean: + // (1) opening delimiter + // (2) stacked verbatim delimiter + if ( + valueCharsArray.length == 0 || + valueCharsArray[valueCharsArray.length - 1] != '\\' + ) { + delim['{'].push(this.CHAR_) + this.SKIPWS_ = false + this.SKIPCOMMENT_ = false + } else { + // literal, add to value + } + } + if (c == '}') { + // This brace can mean: + // (1) closing delimiter + // (2) closing stacked verbatim delimiter + // (3) end of object definition if value was a macro + if (delim['"'].length + delim['{'].length === 0) { + // end of object definition, after macro + const macro = this.PARSETMP_.Value.join('').trim() + if (macro in this.MACROS_) { + // Successful macro reference + this.PARSETMP_.Value = [this.MACROS_[macro]] + } else { + // Reference to an undefined macro + this.error_('Reference to an undefined macro: ' + macro) + } + AnotherIteration = true + doneParsingValue = true + } else { + // sometimes imported bibs will have {\},{\\}, {\\\}, {\\\\}, etc for whitespace, + // which would otherwise break the parsing. we watch for these occurences of + // 1+ backslashes in an empty bracket pair to gracefully handle the malformed bib file + const doubleSlash = + valueCharsArray.length >= 2 && + valueCharsArray[valueCharsArray.length - 1] === '\\' && // for \\} + valueCharsArray[valueCharsArray.length - 2] === '\\' + const singleSlash = + valueCharsArray.length >= 2 && + valueCharsArray[valueCharsArray.length - 1] === '\\' && // for {\} + valueCharsArray[valueCharsArray.length - 2] === '{' + + if ( + valueCharsArray.length == 0 || + valueCharsArray[valueCharsArray.length - 1] != '\\' || // for } + doubleSlash || + singleSlash + ) { + if (delim['{'].length > 0) { + // pop stack for stacked verbatim delimiter + delim['{'].splice(delim['{'].length - 1, 1) + if (delim['{'].length + delim['"'].length == 0) { + // closing delimiter + doneParsingValue = true + } else { + // end verbatim block + } + } + } else { + // literal, add to value + } + } + } + } + + // If here, then we are either done parsing the value or + // have a literal that should be added to the value. + if (doneParsingValue) { + // SUCCESS: value parsed + // NEXT_STATE: KV_KEY + this.SKIPWS_ = true + this.SKIPCOMMENT_ = true + this.STATE_ = this.STATES_.KV_KEY + if (!this.SKIPKVPAIR_) { + this.DATA_.Fields[this.PARSETMP_.Key] = + this.PARSETMP_.Value.join('') + } + this.PARSETMP_ = { Key: '' } + this.VALBRACES_ = null + } else { + this.PARSETMP_.Value.push(c) + if (this.PARSETMP_.Value.length >= 1000 * 20) { + this.PARSETMP_.Value = [] + this.STATE_ = this.STATES_.ENTRY_OR_JUNK + this.DATA_ = { ObjectType: '' } + this.BRACETYPE_ = null + this.SKIPWS_ = true + this.SKIPCOMMENT_ = true + } + } + break + } // end switch (this.STATE_) + } // end while(AnotherIteration) +} // end function processCharacter + +/** @private */ BibtexParser.prototype.CHARCONV_ = [ + [/\\space /g, '\u0020'], + [/\\textdollar /g, '\u0024'], + [/\\textquotesingle /g, '\u0027'], + [/\\ast /g, '\u002A'], + [/\\textbackslash /g, '\u005C'], + [/\\\^\{\}/g, '\u005E'], + [/\\textasciigrave /g, '\u0060'], + [/\\lbrace /g, '\u007B'], + [/\\vert /g, '\u007C'], + [/\\rbrace /g, '\u007D'], + [/\\textasciitilde /g, '\u007E'], + [/\\textexclamdown /g, '\u00A1'], + [/\\textcent /g, '\u00A2'], + [/\\textsterling /g, '\u00A3'], + [/\\textcurrency /g, '\u00A4'], + [/\\textyen /g, '\u00A5'], + [/\\textbrokenbar /g, '\u00A6'], + [/\\textsection /g, '\u00A7'], + [/\\textasciidieresis /g, '\u00A8'], + [/\\textcopyright /g, '\u00A9'], + [/\\textordfeminine /g, '\u00AA'], + [/\\guillemotleft /g, '\u00AB'], + [/\\lnot /g, '\u00AC'], + [/\\textregistered /g, '\u00AE'], + [/\\textasciimacron /g, '\u00AF'], + [/\\textdegree /g, '\u00B0'], + [/\\pm /g, '\u00B1'], + [/\\textasciiacute /g, '\u00B4'], + [/\\mathrm\{\\mu\}/g, '\u00B5'], + [/\\textparagraph /g, '\u00B6'], + [/\\cdot /g, '\u00B7'], + [/\\c\{\}/g, '\u00B8'], + [/\\textordmasculine /g, '\u00BA'], + [/\\guillemotright /g, '\u00BB'], + [/\\textonequarter /g, '\u00BC'], + [/\\textonehalf /g, '\u00BD'], + [/\\textthreequarters /g, '\u00BE'], + [/\\textquestiondown /g, '\u00BF'], + [/\\`\{A\}/g, '\u00C0'], + [/\\'\{A\}/g, '\u00C1'], + [/\\\^\{A\}/g, '\u00C2'], + [/\\~\{A\}/g, '\u00C3'], + [/\\"\{A\}/g, '\u00C4'], + [/\\AA /g, '\u00C5'], + [/\\AE /g, '\u00C6'], + [/\\c\{C\}/g, '\u00C7'], + [/\\`\{E\}/g, '\u00C8'], + [/\\'\{E\}/g, '\u00C9'], + [/\\\^\{E\}/g, '\u00CA'], + [/\\"\{E\}/g, '\u00CB'], + [/\\`\{I\}/g, '\u00CC'], + [/\\'\{I\}/g, '\u00CD'], + [/\\\^\{I\}/g, '\u00CE'], + [/\\"\{I\}/g, '\u00CF'], + [/\\DH /g, '\u00D0'], + [/\\~\{N\}/g, '\u00D1'], + [/\\`\{O\}/g, '\u00D2'], + [/\\'\{O\}/g, '\u00D3'], + [/\\\^\{O\}/g, '\u00D4'], + [/\\~\{O\}/g, '\u00D5'], + [/\\"\{O\}/g, '\u00D6'], + [/\\texttimes /g, '\u00D7'], + [/\\O /g, '\u00D8'], + [/\\`\{U\}/g, '\u00D9'], + [/\\'\{U\}/g, '\u00DA'], + [/\\\^\{U\}/g, '\u00DB'], + [/\\"\{U\}/g, '\u00DC'], + [/\\'\{Y\}/g, '\u00DD'], + [/\\TH /g, '\u00DE'], + [/\\ss /g, '\u00DF'], + [/\\`\{a\}/g, '\u00E0'], + [/\\'\{a\}/g, '\u00E1'], + [/\\\^\{a\}/g, '\u00E2'], + [/\\~\{a\}/g, '\u00E3'], + [/\\"\{a\}/g, '\u00E4'], + [/\\aa /g, '\u00E5'], + [/\\ae /g, '\u00E6'], + [/\\c\{c\}/g, '\u00E7'], + [/\\`\{e\}/g, '\u00E8'], + [/\\'\{e\}/g, '\u00E9'], + [/\\\^\{e\}/g, '\u00EA'], + [/\\"\{e\}/g, '\u00EB'], + [/\\`\{\\i\}/g, '\u00EC'], + [/\\'\{\\i\}/g, '\u00ED'], + [/\\\^\{\\i\}/g, '\u00EE'], + [/\\"\{\\i\}/g, '\u00EF'], + [/\\dh /g, '\u00F0'], + [/\\~\{n\}/g, '\u00F1'], + [/\\`\{o\}/g, '\u00F2'], + [/\\'\{o\}/g, '\u00F3'], + [/\\\^\{o\}/g, '\u00F4'], + [/\\~\{o\}/g, '\u00F5'], + [/\\"\{o\}/g, '\u00F6'], + [/\\div /g, '\u00F7'], + [/\\o /g, '\u00F8'], + [/\\`\{u\}/g, '\u00F9'], + [/\\'\{u\}/g, '\u00FA'], + [/\\\^\{u\}/g, '\u00FB'], + [/\\"\{u\}/g, '\u00FC'], + [/\\'\{y\}/g, '\u00FD'], + [/\\th /g, '\u00FE'], + [/\\"\{y\}/g, '\u00FF'], + [/\\=\{A\}/g, '\u0100'], + [/\\=\{a\}/g, '\u0101'], + [/\\u\{A\}/g, '\u0102'], + [/\\u\{a\}/g, '\u0103'], + [/\\k\{A\}/g, '\u0104'], + [/\\k\{a\}/g, '\u0105'], + [/\\'\{C\}/g, '\u0106'], + [/\\'\{c\}/g, '\u0107'], + [/\\\^\{C\}/g, '\u0108'], + [/\\\^\{c\}/g, '\u0109'], + [/\\.\{C\}/g, '\u010A'], + [/\\.\{c\}/g, '\u010B'], + [/\\v\{C\}/g, '\u010C'], + [/\\v\{c\}/g, '\u010D'], + [/\\v\{D\}/g, '\u010E'], + [/\\v\{d\}/g, '\u010F'], + [/\\DJ /g, '\u0110'], + [/\\dj /g, '\u0111'], + [/\\=\{E\}/g, '\u0112'], + [/\\=\{e\}/g, '\u0113'], + [/\\u\{E\}/g, '\u0114'], + [/\\u\{e\}/g, '\u0115'], + [/\\.\{E\}/g, '\u0116'], + [/\\.\{e\}/g, '\u0117'], + [/\\k\{E\}/g, '\u0118'], + [/\\k\{e\}/g, '\u0119'], + [/\\v\{E\}/g, '\u011A'], + [/\\v\{e\}/g, '\u011B'], + [/\\\^\{G\}/g, '\u011C'], + [/\\\^\{g\}/g, '\u011D'], + [/\\u\{G\}/g, '\u011E'], + [/\\u\{g\}/g, '\u011F'], + [/\\.\{G\}/g, '\u0120'], + [/\\.\{g\}/g, '\u0121'], + [/\\c\{G\}/g, '\u0122'], + [/\\c\{g\}/g, '\u0123'], + [/\\\^\{H\}/g, '\u0124'], + [/\\\^\{h\}/g, '\u0125'], + [/\\Elzxh /g, '\u0127'], + [/\\~\{I\}/g, '\u0128'], + [/\\~\{\\i\}/g, '\u0129'], + [/\\=\{I\}/g, '\u012A'], + [/\\=\{\\i\}/g, '\u012B'], + [/\\u\{I\}/g, '\u012C'], + [/\\u\{\\i\}/g, '\u012D'], + [/\\k\{I\}/g, '\u012E'], + [/\\k\{i\}/g, '\u012F'], + [/\\.\{I\}/g, '\u0130'], + [/\\i /g, '\u0131'], + [/\\\^\{J\}/g, '\u0134'], + [/\\\^\{\\j\}/g, '\u0135'], + [/\\c\{K\}/g, '\u0136'], + [/\\c\{k\}/g, '\u0137'], + [/\\'\{L\}/g, '\u0139'], + [/\\'\{l\}/g, '\u013A'], + [/\\c\{L\}/g, '\u013B'], + [/\\c\{l\}/g, '\u013C'], + [/\\v\{L\}/g, '\u013D'], + [/\\v\{l\}/g, '\u013E'], + [/\\L /g, '\u0141'], + [/\\l /g, '\u0142'], + [/\\'\{N\}/g, '\u0143'], + [/\\'\{n\}/g, '\u0144'], + [/\\c\{N\}/g, '\u0145'], + [/\\c\{n\}/g, '\u0146'], + [/\\v\{N\}/g, '\u0147'], + [/\\v\{n\}/g, '\u0148'], + [/\\NG /g, '\u014A'], + [/\\ng /g, '\u014B'], + [/\\=\{O\}/g, '\u014C'], + [/\\=\{o\}/g, '\u014D'], + [/\\u\{O\}/g, '\u014E'], + [/\\u\{o\}/g, '\u014F'], + [/\\H\{O\}/g, '\u0150'], + [/\\H\{o\}/g, '\u0151'], + [/\\OE /g, '\u0152'], + [/\\oe /g, '\u0153'], + [/\\'\{R\}/g, '\u0154'], + [/\\'\{r\}/g, '\u0155'], + [/\\c\{R\}/g, '\u0156'], + [/\\c\{r\}/g, '\u0157'], + [/\\v\{R\}/g, '\u0158'], + [/\\v\{r\}/g, '\u0159'], + [/\\'\{S\}/g, '\u015A'], + [/\\'\{s\}/g, '\u015B'], + [/\\\^\{S\}/g, '\u015C'], + [/\\\^\{s\}/g, '\u015D'], + [/\\c\{S\}/g, '\u015E'], + [/\\c\{s\}/g, '\u015F'], + [/\\v\{S\}/g, '\u0160'], + [/\\v\{s\}/g, '\u0161'], + [/\\c\{T\}/g, '\u0162'], + [/\\c\{t\}/g, '\u0163'], + [/\\v\{T\}/g, '\u0164'], + [/\\v\{t\}/g, '\u0165'], + [/\\~\{U\}/g, '\u0168'], + [/\\~\{u\}/g, '\u0169'], + [/\\=\{U\}/g, '\u016A'], + [/\\=\{u\}/g, '\u016B'], + [/\\u\{U\}/g, '\u016C'], + [/\\u\{u\}/g, '\u016D'], + [/\\r\{U\}/g, '\u016E'], + [/\\r\{u\}/g, '\u016F'], + [/\\H\{U\}/g, '\u0170'], + [/\\H\{u\}/g, '\u0171'], + [/\\k\{U\}/g, '\u0172'], + [/\\k\{u\}/g, '\u0173'], + [/\\\^\{W\}/g, '\u0174'], + [/\\\^\{w\}/g, '\u0175'], + [/\\\^\{Y\}/g, '\u0176'], + [/\\\^\{y\}/g, '\u0177'], + [/\\"\{Y\}/g, '\u0178'], + [/\\'\{Z\}/g, '\u0179'], + [/\\'\{z\}/g, '\u017A'], + [/\\.\{Z\}/g, '\u017B'], + [/\\.\{z\}/g, '\u017C'], + [/\\v\{Z\}/g, '\u017D'], + [/\\v\{z\}/g, '\u017E'], + [/\\texthvlig /g, '\u0195'], + [/\\textnrleg /g, '\u019E'], + [/\\eth /g, '\u01AA'], + [/\\textdoublepipe /g, '\u01C2'], + [/\\'\{g\}/g, '\u01F5'], + [/\\Elztrna /g, '\u0250'], + [/\\Elztrnsa /g, '\u0252'], + [/\\Elzopeno /g, '\u0254'], + [/\\Elzrtld /g, '\u0256'], + [/\\Elzschwa /g, '\u0259'], + [/\\varepsilon /g, '\u025B'], + [/\\Elzpgamma /g, '\u0263'], + [/\\Elzpbgam /g, '\u0264'], + [/\\Elztrnh /g, '\u0265'], + [/\\Elzbtdl /g, '\u026C'], + [/\\Elzrtll /g, '\u026D'], + [/\\Elztrnm /g, '\u026F'], + [/\\Elztrnmlr /g, '\u0270'], + [/\\Elzltlmr /g, '\u0271'], + [/\\Elzltln /g, '\u0272'], + [/\\Elzrtln /g, '\u0273'], + [/\\Elzclomeg /g, '\u0277'], + [/\\textphi /g, '\u0278'], + [/\\Elztrnr /g, '\u0279'], + [/\\Elztrnrl /g, '\u027A'], + [/\\Elzrttrnr /g, '\u027B'], + [/\\Elzrl /g, '\u027C'], + [/\\Elzrtlr /g, '\u027D'], + [/\\Elzfhr /g, '\u027E'], + [/\\Elzrtls /g, '\u0282'], + [/\\Elzesh /g, '\u0283'], + [/\\Elztrnt /g, '\u0287'], + [/\\Elzrtlt /g, '\u0288'], + [/\\Elzpupsil /g, '\u028A'], + [/\\Elzpscrv /g, '\u028B'], + [/\\Elzinvv /g, '\u028C'], + [/\\Elzinvw /g, '\u028D'], + [/\\Elztrny /g, '\u028E'], + [/\\Elzrtlz /g, '\u0290'], + [/\\Elzyogh /g, '\u0292'], + [/\\Elzglst /g, '\u0294'], + [/\\Elzreglst /g, '\u0295'], + [/\\Elzinglst /g, '\u0296'], + [/\\textturnk /g, '\u029E'], + [/\\Elzdyogh /g, '\u02A4'], + [/\\Elztesh /g, '\u02A7'], + [/\\textasciicaron /g, '\u02C7'], + [/\\Elzverts /g, '\u02C8'], + [/\\Elzverti /g, '\u02CC'], + [/\\Elzlmrk /g, '\u02D0'], + [/\\Elzhlmrk /g, '\u02D1'], + [/\\Elzsbrhr /g, '\u02D2'], + [/\\Elzsblhr /g, '\u02D3'], + [/\\Elzrais /g, '\u02D4'], + [/\\Elzlow /g, '\u02D5'], + [/\\textasciibreve /g, '\u02D8'], + [/\\textperiodcentered /g, '\u02D9'], + [/\\r\{\}/g, '\u02DA'], + [/\\k\{\}/g, '\u02DB'], + [/\\texttildelow /g, '\u02DC'], + [/\\H\{\}/g, '\u02DD'], + [/\\tone\{55\}/g, '\u02E5'], + [/\\tone\{44\}/g, '\u02E6'], + [/\\tone\{33\}/g, '\u02E7'], + [/\\tone\{22\}/g, '\u02E8'], + [/\\tone\{11\}/g, '\u02E9'], + [/\\cyrchar\\C/g, '\u030F'], + [/\\Elzpalh /g, '\u0321'], + [/\\Elzrh /g, '\u0322'], + [/\\Elzsbbrg /g, '\u032A'], + [/\\Elzxl /g, '\u0335'], + [/\\Elzbar /g, '\u0336'], + [/\\'\{A\}/g, '\u0386'], + [/\\'\{E\}/g, '\u0388'], + [/\\'\{H\}/g, '\u0389'], + [/\\'\{\}\{I\}/g, '\u038A'], + [/\\'\{\}O/g, '\u038C'], + [/\\mathrm\{'Y\}/g, '\u038E'], + [/\\mathrm\{'\\Omega\}/g, '\u038F'], + [/\\acute\{\\ddot\{\\iota\}\}/g, '\u0390'], + [/\\Alpha /g, '\u0391'], + [/\\Beta /g, '\u0392'], + [/\\Gamma /g, '\u0393'], + [/\\Delta /g, '\u0394'], + [/\\Epsilon /g, '\u0395'], + [/\\Zeta /g, '\u0396'], + [/\\Eta /g, '\u0397'], + [/\\Theta /g, '\u0398'], + [/\\Iota /g, '\u0399'], + [/\\Kappa /g, '\u039A'], + [/\\Lambda /g, '\u039B'], + [/\\Xi /g, '\u039E'], + [/\\Pi /g, '\u03A0'], + [/\\Rho /g, '\u03A1'], + [/\\Sigma /g, '\u03A3'], + [/\\Tau /g, '\u03A4'], + [/\\Upsilon /g, '\u03A5'], + [/\\Phi /g, '\u03A6'], + [/\\Chi /g, '\u03A7'], + [/\\Psi /g, '\u03A8'], + [/\\Omega /g, '\u03A9'], + [/\\mathrm\{\\ddot\{I\}\}/g, '\u03AA'], + [/\\mathrm\{\\ddot\{Y\}\}/g, '\u03AB'], + [/\\'\{\$\\alpha\$\}/g, '\u03AC'], + [/\\acute\{\\epsilon\}/g, '\u03AD'], + [/\\acute\{\\eta\}/g, '\u03AE'], + [/\\acute\{\\iota\}/g, '\u03AF'], + [/\\acute\{\\ddot\{\\upsilon\}\}/g, '\u03B0'], + [/\\alpha /g, '\u03B1'], + [/\\beta /g, '\u03B2'], + [/\\gamma /g, '\u03B3'], + [/\\delta /g, '\u03B4'], + [/\\epsilon /g, '\u03B5'], + [/\\zeta /g, '\u03B6'], + [/\\eta /g, '\u03B7'], + [/\\texttheta /g, '\u03B8'], + [/\\iota /g, '\u03B9'], + [/\\kappa /g, '\u03BA'], + [/\\lambda /g, '\u03BB'], + [/\\mu /g, '\u03BC'], + [/\\nu /g, '\u03BD'], + [/\\xi /g, '\u03BE'], + [/\\pi /g, '\u03C0'], + [/\\rho /g, '\u03C1'], + [/\\varsigma /g, '\u03C2'], + [/\\sigma /g, '\u03C3'], + [/\\tau /g, '\u03C4'], + [/\\upsilon /g, '\u03C5'], + [/\\varphi /g, '\u03C6'], + [/\\chi /g, '\u03C7'], + [/\\psi /g, '\u03C8'], + [/\\omega /g, '\u03C9'], + [/\\ddot\{\\iota\}/g, '\u03CA'], + [/\\ddot\{\\upsilon\}/g, '\u03CB'], + [/\\'\{o\}/g, '\u03CC'], + [/\\acute\{\\upsilon\}/g, '\u03CD'], + [/\\acute\{\\omega\}/g, '\u03CE'], + [/\\Pisymbol\{ppi022\}\{87\}/g, '\u03D0'], + [/\\textvartheta /g, '\u03D1'], + [/\\Upsilon /g, '\u03D2'], + [/\\phi /g, '\u03D5'], + [/\\varpi /g, '\u03D6'], + [/\\Stigma /g, '\u03DA'], + [/\\Digamma /g, '\u03DC'], + [/\\digamma /g, '\u03DD'], + [/\\Koppa /g, '\u03DE'], + [/\\Sampi /g, '\u03E0'], + [/\\varkappa /g, '\u03F0'], + [/\\varrho /g, '\u03F1'], + [/\\textTheta /g, '\u03F4'], + [/\\backepsilon /g, '\u03F6'], + [/\\cyrchar\\CYRYO /g, '\u0401'], + [/\\cyrchar\\CYRDJE /g, '\u0402'], + [/\\cyrchar\{\\'\\CYRG\}/g, '\u0403'], + [/\\cyrchar\\CYRIE /g, '\u0404'], + [/\\cyrchar\\CYRDZE /g, '\u0405'], + [/\\cyrchar\\CYRII /g, '\u0406'], + [/\\cyrchar\\CYRYI /g, '\u0407'], + [/\\cyrchar\\CYRJE /g, '\u0408'], + [/\\cyrchar\\CYRLJE /g, '\u0409'], + [/\\cyrchar\\CYRNJE /g, '\u040A'], + [/\\cyrchar\\CYRTSHE /g, '\u040B'], + [/\\cyrchar\{\\'\\CYRK\}/g, '\u040C'], + [/\\cyrchar\\CYRUSHRT /g, '\u040E'], + [/\\cyrchar\\CYRDZHE /g, '\u040F'], + [/\\cyrchar\\CYRA /g, '\u0410'], + [/\\cyrchar\\CYRB /g, '\u0411'], + [/\\cyrchar\\CYRV /g, '\u0412'], + [/\\cyrchar\\CYRG /g, '\u0413'], + [/\\cyrchar\\CYRD /g, '\u0414'], + [/\\cyrchar\\CYRE /g, '\u0415'], + [/\\cyrchar\\CYRZH /g, '\u0416'], + [/\\cyrchar\\CYRZ /g, '\u0417'], + [/\\cyrchar\\CYRI /g, '\u0418'], + [/\\cyrchar\\CYRISHRT /g, '\u0419'], + [/\\cyrchar\\CYRK /g, '\u041A'], + [/\\cyrchar\\CYRL /g, '\u041B'], + [/\\cyrchar\\CYRM /g, '\u041C'], + [/\\cyrchar\\CYRN /g, '\u041D'], + [/\\cyrchar\\CYRO /g, '\u041E'], + [/\\cyrchar\\CYRP /g, '\u041F'], + [/\\cyrchar\\CYRR /g, '\u0420'], + [/\\cyrchar\\CYRS /g, '\u0421'], + [/\\cyrchar\\CYRT /g, '\u0422'], + [/\\cyrchar\\CYRU /g, '\u0423'], + [/\\cyrchar\\CYRF /g, '\u0424'], + [/\\cyrchar\\CYRH /g, '\u0425'], + [/\\cyrchar\\CYRC /g, '\u0426'], + [/\\cyrchar\\CYRCH /g, '\u0427'], + [/\\cyrchar\\CYRSH /g, '\u0428'], + [/\\cyrchar\\CYRSHCH /g, '\u0429'], + [/\\cyrchar\\CYRHRDSN /g, '\u042A'], + [/\\cyrchar\\CYRERY /g, '\u042B'], + [/\\cyrchar\\CYRSFTSN /g, '\u042C'], + [/\\cyrchar\\CYREREV /g, '\u042D'], + [/\\cyrchar\\CYRYU /g, '\u042E'], + [/\\cyrchar\\CYRYA /g, '\u042F'], + [/\\cyrchar\\cyra /g, '\u0430'], + [/\\cyrchar\\cyrb /g, '\u0431'], + [/\\cyrchar\\cyrv /g, '\u0432'], + [/\\cyrchar\\cyrg /g, '\u0433'], + [/\\cyrchar\\cyrd /g, '\u0434'], + [/\\cyrchar\\cyre /g, '\u0435'], + [/\\cyrchar\\cyrzh /g, '\u0436'], + [/\\cyrchar\\cyrz /g, '\u0437'], + [/\\cyrchar\\cyri /g, '\u0438'], + [/\\cyrchar\\cyrishrt /g, '\u0439'], + [/\\cyrchar\\cyrk /g, '\u043A'], + [/\\cyrchar\\cyrl /g, '\u043B'], + [/\\cyrchar\\cyrm /g, '\u043C'], + [/\\cyrchar\\cyrn /g, '\u043D'], + [/\\cyrchar\\cyro /g, '\u043E'], + [/\\cyrchar\\cyrp /g, '\u043F'], + [/\\cyrchar\\cyrr /g, '\u0440'], + [/\\cyrchar\\cyrs /g, '\u0441'], + [/\\cyrchar\\cyrt /g, '\u0442'], + [/\\cyrchar\\cyru /g, '\u0443'], + [/\\cyrchar\\cyrf /g, '\u0444'], + [/\\cyrchar\\cyrh /g, '\u0445'], + [/\\cyrchar\\cyrc /g, '\u0446'], + [/\\cyrchar\\cyrch /g, '\u0447'], + [/\\cyrchar\\cyrsh /g, '\u0448'], + [/\\cyrchar\\cyrshch /g, '\u0449'], + [/\\cyrchar\\cyrhrdsn /g, '\u044A'], + [/\\cyrchar\\cyrery /g, '\u044B'], + [/\\cyrchar\\cyrsftsn /g, '\u044C'], + [/\\cyrchar\\cyrerev /g, '\u044D'], + [/\\cyrchar\\cyryu /g, '\u044E'], + [/\\cyrchar\\cyrya /g, '\u044F'], + [/\\cyrchar\\cyryo /g, '\u0451'], + [/\\cyrchar\\cyrdje /g, '\u0452'], + [/\\cyrchar\{\\'\\cyrg\}/g, '\u0453'], + [/\\cyrchar\\cyrie /g, '\u0454'], + [/\\cyrchar\\cyrdze /g, '\u0455'], + [/\\cyrchar\\cyrii /g, '\u0456'], + [/\\cyrchar\\cyryi /g, '\u0457'], + [/\\cyrchar\\cyrje /g, '\u0458'], + [/\\cyrchar\\cyrlje /g, '\u0459'], + [/\\cyrchar\\cyrnje /g, '\u045A'], + [/\\cyrchar\\cyrtshe /g, '\u045B'], + [/\\cyrchar\{\\'\\cyrk\}/g, '\u045C'], + [/\\cyrchar\\cyrushrt /g, '\u045E'], + [/\\cyrchar\\cyrdzhe /g, '\u045F'], + [/\\cyrchar\\CYROMEGA /g, '\u0460'], + [/\\cyrchar\\cyromega /g, '\u0461'], + [/\\cyrchar\\CYRYAT /g, '\u0462'], + [/\\cyrchar\\CYRIOTE /g, '\u0464'], + [/\\cyrchar\\cyriote /g, '\u0465'], + [/\\cyrchar\\CYRLYUS /g, '\u0466'], + [/\\cyrchar\\cyrlyus /g, '\u0467'], + [/\\cyrchar\\CYRIOTLYUS /g, '\u0468'], + [/\\cyrchar\\cyriotlyus /g, '\u0469'], + [/\\cyrchar\\CYRBYUS /g, '\u046A'], + [/\\cyrchar\\CYRIOTBYUS /g, '\u046C'], + [/\\cyrchar\\cyriotbyus /g, '\u046D'], + [/\\cyrchar\\CYRKSI /g, '\u046E'], + [/\\cyrchar\\cyrksi /g, '\u046F'], + [/\\cyrchar\\CYRPSI /g, '\u0470'], + [/\\cyrchar\\cyrpsi /g, '\u0471'], + [/\\cyrchar\\CYRFITA /g, '\u0472'], + [/\\cyrchar\\CYRIZH /g, '\u0474'], + [/\\cyrchar\\CYRUK /g, '\u0478'], + [/\\cyrchar\\cyruk /g, '\u0479'], + [/\\cyrchar\\CYROMEGARND /g, '\u047A'], + [/\\cyrchar\\cyromegarnd /g, '\u047B'], + [/\\cyrchar\\CYROMEGATITLO /g, '\u047C'], + [/\\cyrchar\\cyromegatitlo /g, '\u047D'], + [/\\cyrchar\\CYROT /g, '\u047E'], + [/\\cyrchar\\cyrot /g, '\u047F'], + [/\\cyrchar\\CYRKOPPA /g, '\u0480'], + [/\\cyrchar\\cyrkoppa /g, '\u0481'], + [/\\cyrchar\\cyrthousands /g, '\u0482'], + [/\\cyrchar\\cyrhundredthousands /g, '\u0488'], + [/\\cyrchar\\cyrmillions /g, '\u0489'], + [/\\cyrchar\\CYRSEMISFTSN /g, '\u048C'], + [/\\cyrchar\\cyrsemisftsn /g, '\u048D'], + [/\\cyrchar\\CYRRTICK /g, '\u048E'], + [/\\cyrchar\\cyrrtick /g, '\u048F'], + [/\\cyrchar\\CYRGUP /g, '\u0490'], + [/\\cyrchar\\cyrgup /g, '\u0491'], + [/\\cyrchar\\CYRGHCRS /g, '\u0492'], + [/\\cyrchar\\cyrghcrs /g, '\u0493'], + [/\\cyrchar\\CYRGHK /g, '\u0494'], + [/\\cyrchar\\cyrghk /g, '\u0495'], + [/\\cyrchar\\CYRZHDSC /g, '\u0496'], + [/\\cyrchar\\cyrzhdsc /g, '\u0497'], + [/\\cyrchar\\CYRZDSC /g, '\u0498'], + [/\\cyrchar\\cyrzdsc /g, '\u0499'], + [/\\cyrchar\\CYRKDSC /g, '\u049A'], + [/\\cyrchar\\cyrkdsc /g, '\u049B'], + [/\\cyrchar\\CYRKVCRS /g, '\u049C'], + [/\\cyrchar\\cyrkvcrs /g, '\u049D'], + [/\\cyrchar\\CYRKHCRS /g, '\u049E'], + [/\\cyrchar\\cyrkhcrs /g, '\u049F'], + [/\\cyrchar\\CYRKBEAK /g, '\u04A0'], + [/\\cyrchar\\cyrkbeak /g, '\u04A1'], + [/\\cyrchar\\CYRNDSC /g, '\u04A2'], + [/\\cyrchar\\cyrndsc /g, '\u04A3'], + [/\\cyrchar\\CYRNG /g, '\u04A4'], + [/\\cyrchar\\cyrng /g, '\u04A5'], + [/\\cyrchar\\CYRPHK /g, '\u04A6'], + [/\\cyrchar\\cyrphk /g, '\u04A7'], + [/\\cyrchar\\CYRABHHA /g, '\u04A8'], + [/\\cyrchar\\cyrabhha /g, '\u04A9'], + [/\\cyrchar\\CYRSDSC /g, '\u04AA'], + [/\\cyrchar\\cyrsdsc /g, '\u04AB'], + [/\\cyrchar\\CYRTDSC /g, '\u04AC'], + [/\\cyrchar\\cyrtdsc /g, '\u04AD'], + [/\\cyrchar\\CYRY /g, '\u04AE'], + [/\\cyrchar\\cyry /g, '\u04AF'], + [/\\cyrchar\\CYRYHCRS /g, '\u04B0'], + [/\\cyrchar\\cyryhcrs /g, '\u04B1'], + [/\\cyrchar\\CYRHDSC /g, '\u04B2'], + [/\\cyrchar\\cyrhdsc /g, '\u04B3'], + [/\\cyrchar\\CYRTETSE /g, '\u04B4'], + [/\\cyrchar\\cyrtetse /g, '\u04B5'], + [/\\cyrchar\\CYRCHRDSC /g, '\u04B6'], + [/\\cyrchar\\cyrchrdsc /g, '\u04B7'], + [/\\cyrchar\\CYRCHVCRS /g, '\u04B8'], + [/\\cyrchar\\cyrchvcrs /g, '\u04B9'], + [/\\cyrchar\\CYRSHHA /g, '\u04BA'], + [/\\cyrchar\\cyrshha /g, '\u04BB'], + [/\\cyrchar\\CYRABHCH /g, '\u04BC'], + [/\\cyrchar\\cyrabhch /g, '\u04BD'], + [/\\cyrchar\\CYRABHCHDSC /g, '\u04BE'], + [/\\cyrchar\\cyrabhchdsc /g, '\u04BF'], + [/\\cyrchar\\CYRpalochka /g, '\u04C0'], + [/\\cyrchar\\CYRKHK /g, '\u04C3'], + [/\\cyrchar\\cyrkhk /g, '\u04C4'], + [/\\cyrchar\\CYRNHK /g, '\u04C7'], + [/\\cyrchar\\cyrnhk /g, '\u04C8'], + [/\\cyrchar\\CYRCHLDSC /g, '\u04CB'], + [/\\cyrchar\\cyrchldsc /g, '\u04CC'], + [/\\cyrchar\\CYRAE /g, '\u04D4'], + [/\\cyrchar\\cyrae /g, '\u04D5'], + [/\\cyrchar\\CYRSCHWA /g, '\u04D8'], + [/\\cyrchar\\cyrschwa /g, '\u04D9'], + [/\\cyrchar\\CYRABHDZE /g, '\u04E0'], + [/\\cyrchar\\cyrabhdze /g, '\u04E1'], + [/\\cyrchar\\CYROTLD /g, '\u04E8'], + [/\\cyrchar\\cyrotld /g, '\u04E9'], + [/\\hspace\{0.6em\}/g, '\u2002'], + [/\\hspace\{1em\}/g, '\u2003'], + [/\\hspace\{0.33em\}/g, '\u2004'], + [/\\hspace\{0.25em\}/g, '\u2005'], + [/\\hspace\{0.166em\}/g, '\u2006'], + [/\\hphantom\{0\}/g, '\u2007'], + [/\\hphantom\{,\}/g, '\u2008'], + [/\\hspace\{0.167em\}/g, '\u2009'], + [/\\mkern1mu /g, '\u200A'], + [/\\textendash /g, '\u2013'], + [/\\textemdash /g, '\u2014'], + [/\\rule\{1em\}\{1pt\}/g, '\u2015'], + [/\\Vert /g, '\u2016'], + [/\\Elzreapos /g, '\u201B'], + [/\\textquotedblleft /g, '\u201C'], + [/\\textquotedblright /g, '\u201D'], + [/\\textdagger /g, '\u2020'], + [/\\textdaggerdbl /g, '\u2021'], + [/\\textbullet /g, '\u2022'], + [/\\ldots /g, '\u2026'], + [/\\textperthousand /g, '\u2030'], + [/\\textpertenthousand /g, '\u2031'], + [/\\backprime /g, '\u2035'], + [/\\guilsinglleft /g, '\u2039'], + [/\\guilsinglright /g, '\u203A'], + [/\\mkern4mu /g, '\u205F'], + [/\\nolinebreak /g, '\u2060'], + [/\\ensuremath\{\\Elzpes\}/g, '\u20A7'], + [/\\mbox\{\\texteuro\} /g, '\u20AC'], + [/\\dddot /g, '\u20DB'], + [/\\ddddot /g, '\u20DC'], + [/\\mathbb\{C\}/g, '\u2102'], + [/\\mathscr\{g\}/g, '\u210A'], + [/\\mathscr\{H\}/g, '\u210B'], + [/\\mathfrak\{H\}/g, '\u210C'], + [/\\mathbb\{H\}/g, '\u210D'], + [/\\hslash /g, '\u210F'], + [/\\mathscr\{I\}/g, '\u2110'], + [/\\mathfrak\{I\}/g, '\u2111'], + [/\\mathscr\{L\}/g, '\u2112'], + [/\\mathscr\{l\}/g, '\u2113'], + [/\\mathbb\{N\}/g, '\u2115'], + [/\\cyrchar\\textnumero /g, '\u2116'], + [/\\wp /g, '\u2118'], + [/\\mathbb\{P\}/g, '\u2119'], + [/\\mathbb\{Q\}/g, '\u211A'], + [/\\mathscr\{R\}/g, '\u211B'], + [/\\mathfrak\{R\}/g, '\u211C'], + [/\\mathbb\{R\}/g, '\u211D'], + [/\\Elzxrat /g, '\u211E'], + [/\\texttrademark /g, '\u2122'], + [/\\mathbb\{Z\}/g, '\u2124'], + [/\\Omega /g, '\u2126'], + [/\\mho /g, '\u2127'], + [/\\mathfrak\{Z\}/g, '\u2128'], + [/\\ElsevierGlyph\{2129\}/g, '\u2129'], + [/\\AA /g, '\u212B'], + [/\\mathscr\{B\}/g, '\u212C'], + [/\\mathfrak\{C\}/g, '\u212D'], + [/\\mathscr\{e\}/g, '\u212F'], + [/\\mathscr\{E\}/g, '\u2130'], + [/\\mathscr\{F\}/g, '\u2131'], + [/\\mathscr\{M\}/g, '\u2133'], + [/\\mathscr\{o\}/g, '\u2134'], + [/\\aleph /g, '\u2135'], + [/\\beth /g, '\u2136'], + [/\\gimel /g, '\u2137'], + [/\\daleth /g, '\u2138'], + [/\\textfrac\{1\}\{3\}/g, '\u2153'], + [/\\textfrac\{2\}\{3\}/g, '\u2154'], + [/\\textfrac\{1\}\{5\}/g, '\u2155'], + [/\\textfrac\{2\}\{5\}/g, '\u2156'], + [/\\textfrac\{3\}\{5\}/g, '\u2157'], + [/\\textfrac\{4\}\{5\}/g, '\u2158'], + [/\\textfrac\{1\}\{6\}/g, '\u2159'], + [/\\textfrac\{5\}\{6\}/g, '\u215A'], + [/\\textfrac\{1\}\{8\}/g, '\u215B'], + [/\\textfrac\{3\}\{8\}/g, '\u215C'], + [/\\textfrac\{5\}\{8\}/g, '\u215D'], + [/\\textfrac\{7\}\{8\}/g, '\u215E'], + [/\\leftarrow /g, '\u2190'], + [/\\uparrow /g, '\u2191'], + [/\\rightarrow /g, '\u2192'], + [/\\downarrow /g, '\u2193'], + [/\\leftrightarrow /g, '\u2194'], + [/\\updownarrow /g, '\u2195'], + [/\\nwarrow /g, '\u2196'], + [/\\nearrow /g, '\u2197'], + [/\\searrow /g, '\u2198'], + [/\\swarrow /g, '\u2199'], + [/\\nleftarrow /g, '\u219A'], + [/\\nrightarrow /g, '\u219B'], + [/\\arrowwaveright /g, '\u219C'], + [/\\arrowwaveright /g, '\u219D'], + [/\\twoheadleftarrow /g, '\u219E'], + [/\\twoheadrightarrow /g, '\u21A0'], + [/\\leftarrowtail /g, '\u21A2'], + [/\\rightarrowtail /g, '\u21A3'], + [/\\mapsto /g, '\u21A6'], + [/\\hookleftarrow /g, '\u21A9'], + [/\\hookrightarrow /g, '\u21AA'], + [/\\looparrowleft /g, '\u21AB'], + [/\\looparrowright /g, '\u21AC'], + [/\\leftrightsquigarrow /g, '\u21AD'], + [/\\nleftrightarrow /g, '\u21AE'], + [/\\Lsh /g, '\u21B0'], + [/\\Rsh /g, '\u21B1'], + [/\\ElsevierGlyph\{21B3\}/g, '\u21B3'], + [/\\curvearrowleft /g, '\u21B6'], + [/\\curvearrowright /g, '\u21B7'], + [/\\circlearrowleft /g, '\u21BA'], + [/\\circlearrowright /g, '\u21BB'], + [/\\leftharpoonup /g, '\u21BC'], + [/\\leftharpoondown /g, '\u21BD'], + [/\\upharpoonright /g, '\u21BE'], + [/\\upharpoonleft /g, '\u21BF'], + [/\\rightharpoonup /g, '\u21C0'], + [/\\rightharpoondown /g, '\u21C1'], + [/\\downharpoonright /g, '\u21C2'], + [/\\downharpoonleft /g, '\u21C3'], + [/\\rightleftarrows /g, '\u21C4'], + [/\\dblarrowupdown /g, '\u21C5'], + [/\\leftrightarrows /g, '\u21C6'], + [/\\leftleftarrows /g, '\u21C7'], + [/\\upuparrows /g, '\u21C8'], + [/\\rightrightarrows /g, '\u21C9'], + [/\\downdownarrows /g, '\u21CA'], + [/\\leftrightharpoons /g, '\u21CB'], + [/\\rightleftharpoons /g, '\u21CC'], + [/\\nLeftarrow /g, '\u21CD'], + [/\\nLeftrightarrow /g, '\u21CE'], + [/\\nRightarrow /g, '\u21CF'], + [/\\Leftarrow /g, '\u21D0'], + [/\\Uparrow /g, '\u21D1'], + [/\\Rightarrow /g, '\u21D2'], + [/\\Downarrow /g, '\u21D3'], + [/\\Leftrightarrow /g, '\u21D4'], + [/\\Updownarrow /g, '\u21D5'], + [/\\Lleftarrow /g, '\u21DA'], + [/\\Rrightarrow /g, '\u21DB'], + [/\\rightsquigarrow /g, '\u21DD'], + [/\\DownArrowUpArrow /g, '\u21F5'], + [/\\forall /g, '\u2200'], + [/\\complement /g, '\u2201'], + [/\\partial /g, '\u2202'], + [/\\exists /g, '\u2203'], + [/\\nexists /g, '\u2204'], + [/\\varnothing /g, '\u2205'], + [/\\nabla /g, '\u2207'], + [/\\in /g, '\u2208'], + [/\\not\\in /g, '\u2209'], + [/\\ni /g, '\u220B'], + [/\\not\\ni /g, '\u220C'], + [/\\prod /g, '\u220F'], + [/\\coprod /g, '\u2210'], + [/\\sum /g, '\u2211'], + [/\\mp /g, '\u2213'], + [/\\dotplus /g, '\u2214'], + [/\\setminus /g, '\u2216'], + [/\\circ /g, '\u2218'], + [/\\bullet /g, '\u2219'], + [/\\surd /g, '\u221A'], + [/\\propto /g, '\u221D'], + [/\\infty /g, '\u221E'], + [/\\rightangle /g, '\u221F'], + [/\\angle /g, '\u2220'], + [/\\measuredangle /g, '\u2221'], + [/\\sphericalangle /g, '\u2222'], + [/\\mid /g, '\u2223'], + [/\\nmid /g, '\u2224'], + [/\\parallel /g, '\u2225'], + [/\\nparallel /g, '\u2226'], + [/\\wedge /g, '\u2227'], + [/\\vee /g, '\u2228'], + [/\\cap /g, '\u2229'], + [/\\cup /g, '\u222A'], + [/\\int /g, '\u222B'], + [/\\int\\!\\int /g, '\u222C'], + [/\\int\\!\\int\\!\\int /g, '\u222D'], + [/\\oint /g, '\u222E'], + [/\\surfintegral /g, '\u222F'], + [/\\volintegral /g, '\u2230'], + [/\\clwintegral /g, '\u2231'], + [/\\ElsevierGlyph\{2232\}/g, '\u2232'], + [/\\ElsevierGlyph\{2233\}/g, '\u2233'], + [/\\therefore /g, '\u2234'], + [/\\because /g, '\u2235'], + [/\\Colon /g, '\u2237'], + [/\\ElsevierGlyph\{2238\}/g, '\u2238'], + [/\\mathbin\{\{:\}\\!\\!\{\-\}\\!\\!\{:\}\}/g, '\u223A'], + [/\\homothetic /g, '\u223B'], + [/\\sim /g, '\u223C'], + [/\\backsim /g, '\u223D'], + [/\\lazysinv /g, '\u223E'], + [/\\wr /g, '\u2240'], + [/\\not\\sim /g, '\u2241'], + [/\\ElsevierGlyph\{2242\}/g, '\u2242'], + [/\\NotEqualTilde /g, '\u2242-00338'], + [/\\simeq /g, '\u2243'], + [/\\not\\simeq /g, '\u2244'], + [/\\cong /g, '\u2245'], + [/\\approxnotequal /g, '\u2246'], + [/\\not\\cong /g, '\u2247'], + [/\\approx /g, '\u2248'], + [/\\not\\approx /g, '\u2249'], + [/\\approxeq /g, '\u224A'], + [/\\tildetrpl /g, '\u224B'], + [/\\not\\apid /g, '\u224B-00338'], + [/\\allequal /g, '\u224C'], + [/\\asymp /g, '\u224D'], + [/\\Bumpeq /g, '\u224E'], + [/\\NotHumpDownHump /g, '\u224E-00338'], + [/\\bumpeq /g, '\u224F'], + [/\\NotHumpEqual /g, '\u224F-00338'], + [/\\doteq /g, '\u2250'], + [/\\not\\doteq/g, '\u2250-00338'], + [/\\doteqdot /g, '\u2251'], + [/\\fallingdotseq /g, '\u2252'], + [/\\risingdotseq /g, '\u2253'], + [/\\eqcirc /g, '\u2256'], + [/\\circeq /g, '\u2257'], + [/\\estimates /g, '\u2259'], + [/\\ElsevierGlyph\{225A\}/g, '\u225A'], + [/\\starequal /g, '\u225B'], + [/\\triangleq /g, '\u225C'], + [/\\ElsevierGlyph\{225F\}/g, '\u225F'], + [/\\not =/g, '\u2260'], + [/\\equiv /g, '\u2261'], + [/\\not\\equiv /g, '\u2262'], + [/\\leq /g, '\u2264'], + [/\\geq /g, '\u2265'], + [/\\leqq /g, '\u2266'], + [/\\geqq /g, '\u2267'], + [/\\lneqq /g, '\u2268'], + [/\\lvertneqq /g, '\u2268-0FE00'], + [/\\gneqq /g, '\u2269'], + [/\\gvertneqq /g, '\u2269-0FE00'], + [/\\ll /g, '\u226A'], + [/\\NotLessLess /g, '\u226A-00338'], + [/\\gg /g, '\u226B'], + [/\\NotGreaterGreater /g, '\u226B-00338'], + [/\\between /g, '\u226C'], + [/\\not\\kern\-0.3em\\times /g, '\u226D'], + [/\\not/g, '\u226F'], + [/\\not\\leq /g, '\u2270'], + [/\\not\\geq /g, '\u2271'], + [/\\lessequivlnt /g, '\u2272'], + [/\\greaterequivlnt /g, '\u2273'], + [/\\ElsevierGlyph\{2274\}/g, '\u2274'], + [/\\ElsevierGlyph\{2275\}/g, '\u2275'], + [/\\lessgtr /g, '\u2276'], + [/\\gtrless /g, '\u2277'], + [/\\notlessgreater /g, '\u2278'], + [/\\notgreaterless /g, '\u2279'], + [/\\prec /g, '\u227A'], + [/\\succ /g, '\u227B'], + [/\\preccurlyeq /g, '\u227C'], + [/\\succcurlyeq /g, '\u227D'], + [/\\precapprox /g, '\u227E'], + [/\\NotPrecedesTilde /g, '\u227E-00338'], + [/\\succapprox /g, '\u227F'], + [/\\NotSucceedsTilde /g, '\u227F-00338'], + [/\\not\\prec /g, '\u2280'], + [/\\not\\succ /g, '\u2281'], + [/\\subset /g, '\u2282'], + [/\\supset /g, '\u2283'], + [/\\not\\subset /g, '\u2284'], + [/\\not\\supset /g, '\u2285'], + [/\\subseteq /g, '\u2286'], + [/\\supseteq /g, '\u2287'], + [/\\not\\subseteq /g, '\u2288'], + [/\\not\\supseteq /g, '\u2289'], + [/\\subsetneq /g, '\u228A'], + [/\\varsubsetneqq /g, '\u228A-0FE00'], + [/\\supsetneq /g, '\u228B'], + [/\\varsupsetneq /g, '\u228B-0FE00'], + [/\\uplus /g, '\u228E'], + [/\\sqsubset /g, '\u228F'], + [/\\NotSquareSubset /g, '\u228F-00338'], + [/\\sqsupset /g, '\u2290'], + [/\\NotSquareSuperset /g, '\u2290-00338'], + [/\\sqsubseteq /g, '\u2291'], + [/\\sqsupseteq /g, '\u2292'], + [/\\sqcap /g, '\u2293'], + [/\\sqcup /g, '\u2294'], + [/\\oplus /g, '\u2295'], + [/\\ominus /g, '\u2296'], + [/\\otimes /g, '\u2297'], + [/\\oslash /g, '\u2298'], + [/\\odot /g, '\u2299'], + [/\\circledcirc /g, '\u229A'], + [/\\circledast /g, '\u229B'], + [/\\circleddash /g, '\u229D'], + [/\\boxplus /g, '\u229E'], + [/\\boxminus /g, '\u229F'], + [/\\boxtimes /g, '\u22A0'], + [/\\boxdot /g, '\u22A1'], + [/\\vdash /g, '\u22A2'], + [/\\dashv /g, '\u22A3'], + [/\\top /g, '\u22A4'], + [/\\perp /g, '\u22A5'], + [/\\truestate /g, '\u22A7'], + [/\\forcesextra /g, '\u22A8'], + [/\\Vdash /g, '\u22A9'], + [/\\Vvdash /g, '\u22AA'], + [/\\VDash /g, '\u22AB'], + [/\\nvdash /g, '\u22AC'], + [/\\nvDash /g, '\u22AD'], + [/\\nVdash /g, '\u22AE'], + [/\\nVDash /g, '\u22AF'], + [/\\vartriangleleft /g, '\u22B2'], + [/\\vartriangleright /g, '\u22B3'], + [/\\trianglelefteq /g, '\u22B4'], + [/\\trianglerighteq /g, '\u22B5'], + [/\\original /g, '\u22B6'], + [/\\image /g, '\u22B7'], + [/\\multimap /g, '\u22B8'], + [/\\hermitconjmatrix /g, '\u22B9'], + [/\\intercal /g, '\u22BA'], + [/\\veebar /g, '\u22BB'], + [/\\rightanglearc /g, '\u22BE'], + [/\\ElsevierGlyph\{22C0\}/g, '\u22C0'], + [/\\ElsevierGlyph\{22C1\}/g, '\u22C1'], + [/\\bigcap /g, '\u22C2'], + [/\\bigcup /g, '\u22C3'], + [/\\diamond /g, '\u22C4'], + [/\\cdot /g, '\u22C5'], + [/\\star /g, '\u22C6'], + [/\\divideontimes /g, '\u22C7'], + [/\\bowtie /g, '\u22C8'], + [/\\ltimes /g, '\u22C9'], + [/\\rtimes /g, '\u22CA'], + [/\\leftthreetimes /g, '\u22CB'], + [/\\rightthreetimes /g, '\u22CC'], + [/\\backsimeq /g, '\u22CD'], + [/\\curlyvee /g, '\u22CE'], + [/\\curlywedge /g, '\u22CF'], + [/\\Subset /g, '\u22D0'], + [/\\Supset /g, '\u22D1'], + [/\\Cap /g, '\u22D2'], + [/\\Cup /g, '\u22D3'], + [/\\pitchfork /g, '\u22D4'], + [/\\lessdot /g, '\u22D6'], + [/\\gtrdot /g, '\u22D7'], + [/\\verymuchless /g, '\u22D8'], + [/\\verymuchgreater /g, '\u22D9'], + [/\\lesseqgtr /g, '\u22DA'], + [/\\gtreqless /g, '\u22DB'], + [/\\curlyeqprec /g, '\u22DE'], + [/\\curlyeqsucc /g, '\u22DF'], + [/\\not\\sqsubseteq /g, '\u22E2'], + [/\\not\\sqsupseteq /g, '\u22E3'], + [/\\Elzsqspne /g, '\u22E5'], + [/\\lnsim /g, '\u22E6'], + [/\\gnsim /g, '\u22E7'], + [/\\precedesnotsimilar /g, '\u22E8'], + [/\\succnsim /g, '\u22E9'], + [/\\ntriangleleft /g, '\u22EA'], + [/\\ntriangleright /g, '\u22EB'], + [/\\ntrianglelefteq /g, '\u22EC'], + [/\\ntrianglerighteq /g, '\u22ED'], + [/\\vdots /g, '\u22EE'], + [/\\cdots /g, '\u22EF'], + [/\\upslopeellipsis /g, '\u22F0'], + [/\\downslopeellipsis /g, '\u22F1'], + [/\\barwedge /g, '\u2305'], + [/\\perspcorrespond /g, '\u2306'], + [/\\lceil /g, '\u2308'], + [/\\rceil /g, '\u2309'], + [/\\lfloor /g, '\u230A'], + [/\\rfloor /g, '\u230B'], + [/\\recorder /g, '\u2315'], + [/\\mathchar"2208/g, '\u2316'], + [/\\ulcorner /g, '\u231C'], + [/\\urcorner /g, '\u231D'], + [/\\llcorner /g, '\u231E'], + [/\\lrcorner /g, '\u231F'], + [/\\frown /g, '\u2322'], + [/\\smile /g, '\u2323'], + [/\\langle /g, '\u2329'], + [/\\rangle /g, '\u232A'], + [/\\ElsevierGlyph\{E838\}/g, '\u233D'], + [/\\Elzdlcorn /g, '\u23A3'], + [/\\lmoustache /g, '\u23B0'], + [/\\rmoustache /g, '\u23B1'], + [/\\textvisiblespace /g, '\u2423'], + [/\\ding\{172\}/g, '\u2460'], + [/\\ding\{173\}/g, '\u2461'], + [/\\ding\{174\}/g, '\u2462'], + [/\\ding\{175\}/g, '\u2463'], + [/\\ding\{176\}/g, '\u2464'], + [/\\ding\{177\}/g, '\u2465'], + [/\\ding\{178\}/g, '\u2466'], + [/\\ding\{179\}/g, '\u2467'], + [/\\ding\{180\}/g, '\u2468'], + [/\\ding\{181\}/g, '\u2469'], + [/\\circledS /g, '\u24C8'], + [/\\Elzdshfnc /g, '\u2506'], + [/\\Elzsqfnw /g, '\u2519'], + [/\\diagup /g, '\u2571'], + [/\\ding\{110\}/g, '\u25A0'], + [/\\square /g, '\u25A1'], + [/\\blacksquare /g, '\u25AA'], + [/\\fbox\{~~\}/g, '\u25AD'], + [/\\Elzvrecto /g, '\u25AF'], + [/\\ElsevierGlyph\{E381\}/g, '\u25B1'], + [/\\ding\{115\}/g, '\u25B2'], + [/\\bigtriangleup /g, '\u25B3'], + [/\\blacktriangle /g, '\u25B4'], + [/\\vartriangle /g, '\u25B5'], + [/\\blacktriangleright /g, '\u25B8'], + [/\\triangleright /g, '\u25B9'], + [/\\ding\{116\}/g, '\u25BC'], + [/\\bigtriangledown /g, '\u25BD'], + [/\\blacktriangledown /g, '\u25BE'], + [/\\triangledown /g, '\u25BF'], + [/\\blacktriangleleft /g, '\u25C2'], + [/\\triangleleft /g, '\u25C3'], + [/\\ding\{117\}/g, '\u25C6'], + [/\\lozenge /g, '\u25CA'], + [/\\bigcirc /g, '\u25CB'], + [/\\ding\{108\}/g, '\u25CF'], + [/\\Elzcirfl /g, '\u25D0'], + [/\\Elzcirfr /g, '\u25D1'], + [/\\Elzcirfb /g, '\u25D2'], + [/\\ding\{119\}/g, '\u25D7'], + [/\\Elzrvbull /g, '\u25D8'], + [/\\Elzsqfl /g, '\u25E7'], + [/\\Elzsqfr /g, '\u25E8'], + [/\\Elzsqfse /g, '\u25EA'], + [/\\bigcirc /g, '\u25EF'], + [/\\ding\{72\}/g, '\u2605'], + [/\\ding\{73\}/g, '\u2606'], + [/\\ding\{37\}/g, '\u260E'], + [/\\ding\{42\}/g, '\u261B'], + [/\\ding\{43\}/g, '\u261E'], + [/\\rightmoon /g, '\u263E'], + [/\\mercury /g, '\u263F'], + [/\\venus /g, '\u2640'], + [/\\male /g, '\u2642'], + [/\\jupiter /g, '\u2643'], + [/\\saturn /g, '\u2644'], + [/\\uranus /g, '\u2645'], + [/\\neptune /g, '\u2646'], + [/\\pluto /g, '\u2647'], + [/\\aries /g, '\u2648'], + [/\\taurus /g, '\u2649'], + [/\\gemini /g, '\u264A'], + [/\\cancer /g, '\u264B'], + [/\\leo /g, '\u264C'], + [/\\virgo /g, '\u264D'], + [/\\libra /g, '\u264E'], + [/\\scorpio /g, '\u264F'], + [/\\sagittarius /g, '\u2650'], + [/\\capricornus /g, '\u2651'], + [/\\aquarius /g, '\u2652'], + [/\\pisces /g, '\u2653'], + [/\\ding\{171\}/g, '\u2660'], + [/\\diamond /g, '\u2662'], + [/\\ding\{168\}/g, '\u2663'], + [/\\ding\{170\}/g, '\u2665'], + [/\\ding\{169\}/g, '\u2666'], + [/\\quarternote /g, '\u2669'], + [/\\eighthnote /g, '\u266A'], + [/\\flat /g, '\u266D'], + [/\\natural /g, '\u266E'], + [/\\sharp /g, '\u266F'], + [/\\ding\{33\}/g, '\u2701'], + [/\\ding\{34\}/g, '\u2702'], + [/\\ding\{35\}/g, '\u2703'], + [/\\ding\{36\}/g, '\u2704'], + [/\\ding\{38\}/g, '\u2706'], + [/\\ding\{39\}/g, '\u2707'], + [/\\ding\{40\}/g, '\u2708'], + [/\\ding\{41\}/g, '\u2709'], + [/\\ding\{44\}/g, '\u270C'], + [/\\ding\{45\}/g, '\u270D'], + [/\\ding\{46\}/g, '\u270E'], + [/\\ding\{47\}/g, '\u270F'], + [/\\ding\{48\}/g, '\u2710'], + [/\\ding\{49\}/g, '\u2711'], + [/\\ding\{50\}/g, '\u2712'], + [/\\ding\{51\}/g, '\u2713'], + [/\\ding\{52\}/g, '\u2714'], + [/\\ding\{53\}/g, '\u2715'], + [/\\ding\{54\}/g, '\u2716'], + [/\\ding\{55\}/g, '\u2717'], + [/\\ding\{56\}/g, '\u2718'], + [/\\ding\{57\}/g, '\u2719'], + [/\\ding\{58\}/g, '\u271A'], + [/\\ding\{59\}/g, '\u271B'], + [/\\ding\{60\}/g, '\u271C'], + [/\\ding\{61\}/g, '\u271D'], + [/\\ding\{62\}/g, '\u271E'], + [/\\ding\{63\}/g, '\u271F'], + [/\\ding\{64\}/g, '\u2720'], + [/\\ding\{65\}/g, '\u2721'], + [/\\ding\{66\}/g, '\u2722'], + [/\\ding\{67\}/g, '\u2723'], + [/\\ding\{68\}/g, '\u2724'], + [/\\ding\{69\}/g, '\u2725'], + [/\\ding\{70\}/g, '\u2726'], + [/\\ding\{71\}/g, '\u2727'], + [/\\ding\{73\}/g, '\u2729'], + [/\\ding\{74\}/g, '\u272A'], + [/\\ding\{75\}/g, '\u272B'], + [/\\ding\{76\}/g, '\u272C'], + [/\\ding\{77\}/g, '\u272D'], + [/\\ding\{78\}/g, '\u272E'], + [/\\ding\{79\}/g, '\u272F'], + [/\\ding\{80\}/g, '\u2730'], + [/\\ding\{81\}/g, '\u2731'], + [/\\ding\{82\}/g, '\u2732'], + [/\\ding\{83\}/g, '\u2733'], + [/\\ding\{84\}/g, '\u2734'], + [/\\ding\{85\}/g, '\u2735'], + [/\\ding\{86\}/g, '\u2736'], + [/\\ding\{87\}/g, '\u2737'], + [/\\ding\{88\}/g, '\u2738'], + [/\\ding\{89\}/g, '\u2739'], + [/\\ding\{90\}/g, '\u273A'], + [/\\ding\{91\}/g, '\u273B'], + [/\\ding\{92\}/g, '\u273C'], + [/\\ding\{93\}/g, '\u273D'], + [/\\ding\{94\}/g, '\u273E'], + [/\\ding\{95\}/g, '\u273F'], + [/\\ding\{96\}/g, '\u2740'], + [/\\ding\{97\}/g, '\u2741'], + [/\\ding\{98\}/g, '\u2742'], + [/\\ding\{99\}/g, '\u2743'], + [/\\ding\{100\}/g, '\u2744'], + [/\\ding\{101\}/g, '\u2745'], + [/\\ding\{102\}/g, '\u2746'], + [/\\ding\{103\}/g, '\u2747'], + [/\\ding\{104\}/g, '\u2748'], + [/\\ding\{105\}/g, '\u2749'], + [/\\ding\{106\}/g, '\u274A'], + [/\\ding\{107\}/g, '\u274B'], + [/\\ding\{109\}/g, '\u274D'], + [/\\ding\{111\}/g, '\u274F'], + [/\\ding\{112\}/g, '\u2750'], + [/\\ding\{113\}/g, '\u2751'], + [/\\ding\{114\}/g, '\u2752'], + [/\\ding\{118\}/g, '\u2756'], + [/\\ding\{120\}/g, '\u2758'], + [/\\ding\{121\}/g, '\u2759'], + [/\\ding\{122\}/g, '\u275A'], + [/\\ding\{123\}/g, '\u275B'], + [/\\ding\{124\}/g, '\u275C'], + [/\\ding\{125\}/g, '\u275D'], + [/\\ding\{126\}/g, '\u275E'], + [/\\ding\{161\}/g, '\u2761'], + [/\\ding\{162\}/g, '\u2762'], + [/\\ding\{163\}/g, '\u2763'], + [/\\ding\{164\}/g, '\u2764'], + [/\\ding\{165\}/g, '\u2765'], + [/\\ding\{166\}/g, '\u2766'], + [/\\ding\{167\}/g, '\u2767'], + [/\\ding\{182\}/g, '\u2776'], + [/\\ding\{183\}/g, '\u2777'], + [/\\ding\{184\}/g, '\u2778'], + [/\\ding\{185\}/g, '\u2779'], + [/\\ding\{186\}/g, '\u277A'], + [/\\ding\{187\}/g, '\u277B'], + [/\\ding\{188\}/g, '\u277C'], + [/\\ding\{189\}/g, '\u277D'], + [/\\ding\{190\}/g, '\u277E'], + [/\\ding\{191\}/g, '\u277F'], + [/\\ding\{192\}/g, '\u2780'], + [/\\ding\{193\}/g, '\u2781'], + [/\\ding\{194\}/g, '\u2782'], + [/\\ding\{195\}/g, '\u2783'], + [/\\ding\{196\}/g, '\u2784'], + [/\\ding\{197\}/g, '\u2785'], + [/\\ding\{198\}/g, '\u2786'], + [/\\ding\{199\}/g, '\u2787'], + [/\\ding\{200\}/g, '\u2788'], + [/\\ding\{201\}/g, '\u2789'], + [/\\ding\{202\}/g, '\u278A'], + [/\\ding\{203\}/g, '\u278B'], + [/\\ding\{204\}/g, '\u278C'], + [/\\ding\{205\}/g, '\u278D'], + [/\\ding\{206\}/g, '\u278E'], + [/\\ding\{207\}/g, '\u278F'], + [/\\ding\{208\}/g, '\u2790'], + [/\\ding\{209\}/g, '\u2791'], + [/\\ding\{210\}/g, '\u2792'], + [/\\ding\{211\}/g, '\u2793'], + [/\\ding\{212\}/g, '\u2794'], + [/\\ding\{216\}/g, '\u2798'], + [/\\ding\{217\}/g, '\u2799'], + [/\\ding\{218\}/g, '\u279A'], + [/\\ding\{219\}/g, '\u279B'], + [/\\ding\{220\}/g, '\u279C'], + [/\\ding\{221\}/g, '\u279D'], + [/\\ding\{222\}/g, '\u279E'], + [/\\ding\{223\}/g, '\u279F'], + [/\\ding\{224\}/g, '\u27A0'], + [/\\ding\{225\}/g, '\u27A1'], + [/\\ding\{226\}/g, '\u27A2'], + [/\\ding\{227\}/g, '\u27A3'], + [/\\ding\{228\}/g, '\u27A4'], + [/\\ding\{229\}/g, '\u27A5'], + [/\\ding\{230\}/g, '\u27A6'], + [/\\ding\{231\}/g, '\u27A7'], + [/\\ding\{232\}/g, '\u27A8'], + [/\\ding\{233\}/g, '\u27A9'], + [/\\ding\{234\}/g, '\u27AA'], + [/\\ding\{235\}/g, '\u27AB'], + [/\\ding\{236\}/g, '\u27AC'], + [/\\ding\{237\}/g, '\u27AD'], + [/\\ding\{238\}/g, '\u27AE'], + [/\\ding\{239\}/g, '\u27AF'], + [/\\ding\{241\}/g, '\u27B1'], + [/\\ding\{242\}/g, '\u27B2'], + [/\\ding\{243\}/g, '\u27B3'], + [/\\ding\{244\}/g, '\u27B4'], + [/\\ding\{245\}/g, '\u27B5'], + [/\\ding\{246\}/g, '\u27B6'], + [/\\ding\{247\}/g, '\u27B7'], + [/\\ding\{248\}/g, '\u27B8'], + [/\\ding\{249\}/g, '\u27B9'], + [/\\ding\{250\}/g, '\u27BA'], + [/\\ding\{251\}/g, '\u27BB'], + [/\\ding\{252\}/g, '\u27BC'], + [/\\ding\{253\}/g, '\u27BD'], + [/\\ding\{254\}/g, '\u27BE'], + [/\\longleftarrow /g, '\u27F5'], + [/\\longrightarrow /g, '\u27F6'], + [/\\longleftrightarrow /g, '\u27F7'], + [/\\Longleftarrow /g, '\u27F8'], + [/\\Longrightarrow /g, '\u27F9'], + [/\\Longleftrightarrow /g, '\u27FA'], + [/\\longmapsto /g, '\u27FC'], + [/\\sim\\joinrel\\leadsto/g, '\u27FF'], + [/\\ElsevierGlyph\{E212\}/g, '\u2905'], + [/\\UpArrowBar /g, '\u2912'], + [/\\DownArrowBar /g, '\u2913'], + [/\\ElsevierGlyph\{E20C\}/g, '\u2923'], + [/\\ElsevierGlyph\{E20D\}/g, '\u2924'], + [/\\ElsevierGlyph\{E20B\}/g, '\u2925'], + [/\\ElsevierGlyph\{E20A\}/g, '\u2926'], + [/\\ElsevierGlyph\{E211\}/g, '\u2927'], + [/\\ElsevierGlyph\{E20E\}/g, '\u2928'], + [/\\ElsevierGlyph\{E20F\}/g, '\u2929'], + [/\\ElsevierGlyph\{E210\}/g, '\u292A'], + [/\\ElsevierGlyph\{E21C\}/g, '\u2933'], + [/\\ElsevierGlyph\{E21D\}/g, '\u2933-00338'], + [/\\ElsevierGlyph\{E21A\}/g, '\u2936'], + [/\\ElsevierGlyph\{E219\}/g, '\u2937'], + [/\\Elolarr /g, '\u2940'], + [/\\Elorarr /g, '\u2941'], + [/\\ElzRlarr /g, '\u2942'], + [/\\ElzrLarr /g, '\u2944'], + [/\\Elzrarrx /g, '\u2947'], + [/\\LeftRightVector /g, '\u294E'], + [/\\RightUpDownVector /g, '\u294F'], + [/\\DownLeftRightVector /g, '\u2950'], + [/\\LeftUpDownVector /g, '\u2951'], + [/\\LeftVectorBar /g, '\u2952'], + [/\\RightVectorBar /g, '\u2953'], + [/\\RightUpVectorBar /g, '\u2954'], + [/\\RightDownVectorBar /g, '\u2955'], + [/\\DownLeftVectorBar /g, '\u2956'], + [/\\DownRightVectorBar /g, '\u2957'], + [/\\LeftUpVectorBar /g, '\u2958'], + [/\\LeftDownVectorBar /g, '\u2959'], + [/\\LeftTeeVector /g, '\u295A'], + [/\\RightTeeVector /g, '\u295B'], + [/\\RightUpTeeVector /g, '\u295C'], + [/\\RightDownTeeVector /g, '\u295D'], + [/\\DownLeftTeeVector /g, '\u295E'], + [/\\DownRightTeeVector /g, '\u295F'], + [/\\LeftUpTeeVector /g, '\u2960'], + [/\\LeftDownTeeVector /g, '\u2961'], + [/\\UpEquilibrium /g, '\u296E'], + [/\\ReverseUpEquilibrium /g, '\u296F'], + [/\\RoundImplies /g, '\u2970'], + [/\\ElsevierGlyph\{E214\}/g, '\u297C'], + [/\\ElsevierGlyph\{E215\}/g, '\u297D'], + [/\\Elztfnc /g, '\u2980'], + [/\\ElsevierGlyph\{3018\}/g, '\u2985'], + [/\\Elroang /g, '\u2986'], + [/\\ElsevierGlyph\{E291\}/g, '\u2994'], + [/\\Elzddfnc /g, '\u2999'], + [/\\Angle /g, '\u299C'], + [/\\Elzlpargt /g, '\u29A0'], + [/\\ElsevierGlyph\{E260\}/g, '\u29B5'], + [/\\ElsevierGlyph\{E61B\}/g, '\u29B6'], + [/\\ElzLap /g, '\u29CA'], + [/\\Elzdefas /g, '\u29CB'], + [/\\LeftTriangleBar /g, '\u29CF'], + [/\\NotLeftTriangleBar /g, '\u29CF-00338'], + [/\\RightTriangleBar /g, '\u29D0'], + [/\\NotRightTriangleBar /g, '\u29D0-00338'], + [/\\ElsevierGlyph\{E372\}/g, '\u29DC'], + [/\\blacklozenge /g, '\u29EB'], + [/\\RuleDelayed /g, '\u29F4'], + [/\\Elxuplus /g, '\u2A04'], + [/\\ElzThr /g, '\u2A05'], + [/\\Elxsqcup /g, '\u2A06'], + [/\\ElzInf /g, '\u2A07'], + [/\\ElzSup /g, '\u2A08'], + [/\\ElzCint /g, '\u2A0D'], + [/\\clockoint /g, '\u2A0F'], + [/\\ElsevierGlyph\{E395\}/g, '\u2A10'], + [/\\sqrint /g, '\u2A16'], + [/\\ElsevierGlyph\{E25A\}/g, '\u2A25'], + [/\\ElsevierGlyph\{E25B\}/g, '\u2A2A'], + [/\\ElsevierGlyph\{E25C\}/g, '\u2A2D'], + [/\\ElsevierGlyph\{E25D\}/g, '\u2A2E'], + [/\\ElzTimes /g, '\u2A2F'], + [/\\ElsevierGlyph\{E25E\}/g, '\u2A34'], + [/\\ElsevierGlyph\{E25E\}/g, '\u2A35'], + [/\\ElsevierGlyph\{E259\}/g, '\u2A3C'], + [/\\amalg /g, '\u2A3F'], + [/\\ElzAnd /g, '\u2A53'], + [/\\ElzOr /g, '\u2A54'], + [/\\ElsevierGlyph\{E36E\}/g, '\u2A55'], + [/\\ElOr /g, '\u2A56'], + [/\\perspcorrespond /g, '\u2A5E'], + [/\\Elzminhat /g, '\u2A5F'], + [/\\ElsevierGlyph\{225A\}/g, '\u2A63'], + [/\\stackrel\{*\}\{=\}/g, '\u2A6E'], + [/\\Equal /g, '\u2A75'], + [/\\leqslant /g, '\u2A7D'], + [/\\nleqslant /g, '\u2A7D-00338'], + [/\\geqslant /g, '\u2A7E'], + [/\\ngeqslant /g, '\u2A7E-00338'], + [/\\lessapprox /g, '\u2A85'], + [/\\gtrapprox /g, '\u2A86'], + [/\\lneq /g, '\u2A87'], + [/\\gneq /g, '\u2A88'], + [/\\lnapprox /g, '\u2A89'], + [/\\gnapprox /g, '\u2A8A'], + [/\\lesseqqgtr /g, '\u2A8B'], + [/\\gtreqqless /g, '\u2A8C'], + [/\\eqslantless /g, '\u2A95'], + [/\\eqslantgtr /g, '\u2A96'], + [/\\Pisymbol\{ppi020\}\{117\}/g, '\u2A9D'], + [/\\Pisymbol\{ppi020\}\{105\}/g, '\u2A9E'], + [/\\NestedLessLess /g, '\u2AA1'], + [/\\NotNestedLessLess /g, '\u2AA1-00338'], + [/\\NestedGreaterGreater /g, '\u2AA2'], + [/\\NotNestedGreaterGreater /g, '\u2AA2-00338'], + [/\\preceq /g, '\u2AAF'], + [/\\not\\preceq /g, '\u2AAF-00338'], + [/\\succeq /g, '\u2AB0'], + [/\\not\\succeq /g, '\u2AB0-00338'], + [/\\precneqq /g, '\u2AB5'], + [/\\succneqq /g, '\u2AB6'], + [/\\precapprox /g, '\u2AB7'], + [/\\succapprox /g, '\u2AB8'], + [/\\precnapprox /g, '\u2AB9'], + [/\\succnapprox /g, '\u2ABA'], + [/\\subseteqq /g, '\u2AC5'], + [/\\nsubseteqq /g, '\u2AC5-00338'], + [/\\supseteqq /g, '\u2AC6'], + [/\\nsupseteqq/g, '\u2AC6-00338'], + [/\\subsetneqq /g, '\u2ACB'], + [/\\supsetneqq /g, '\u2ACC'], + [/\\ElsevierGlyph\{E30D\}/g, '\u2AEB'], + [/\\Elztdcol /g, '\u2AF6'], + [/\\ElsevierGlyph\{300A\}/g, '\u300A'], + [/\\ElsevierGlyph\{300B\}/g, '\u300B'], + [/\\ElsevierGlyph\{3018\}/g, '\u3018'], + [/\\ElsevierGlyph\{3019\}/g, '\u3019'], + [/\\openbracketleft /g, '\u301A'], + [/\\openbracketright /g, '\u301B'], +] + +export default BibtexParser +if (typeof module !== 'undefined' && module.exports) { + module.exports = BibtexParser +} diff --git a/services/references/buildscript.txt b/services/references/buildscript.txt new file mode 100644 index 00000000000..324691a60b4 --- /dev/null +++ b/services/references/buildscript.txt @@ -0,0 +1,6 @@ +references +--dependencies= +--env-add= +--env-pass-through= +--node-version=22.18.0 +--public-repo=False diff --git a/services/references/config/settings.defaults.cjs b/services/references/config/settings.defaults.cjs new file mode 100644 index 00000000000..2551f99f096 --- /dev/null +++ b/services/references/config/settings.defaults.cjs @@ -0,0 +1,9 @@ +module.exports = { + internal: { + references: { + port: 3056, + host: process.env.REFERENCES_HOST || '127.0.0.1', + }, + }, +} + diff --git a/services/references/docker-compose.ci.yml b/services/references/docker-compose.ci.yml new file mode 100644 index 00000000000..9c93e99a355 --- /dev/null +++ b/services/references/docker-compose.ci.yml @@ -0,0 +1,38 @@ +# This file was auto-generated, do not edit it directly. +# Instead run bin/update_build_scripts from +# https://github.com/overleaf/internal/ + +services: + test_unit: + image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER + user: node + command: npm run test:unit:_run + environment: + NODE_ENV: test + NODE_OPTIONS: "--unhandled-rejections=strict" + + + test_acceptance: + build: . + image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER + environment: + ELASTIC_SEARCH_DSN: es:9200 + MONGO_HOST: mongo + POSTGRES_HOST: postgres + MOCHA_GREP: ${MOCHA_GREP} + NODE_ENV: test + NODE_OPTIONS: "--unhandled-rejections=strict" + depends_on: + mongo: + condition: service_started + user: node + command: npm run test:acceptance + + + tar: + build: . + image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER + volumes: + - ./:/tmp/build/ + command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs . + user: root diff --git a/services/references/docker-compose.yml b/services/references/docker-compose.yml new file mode 100644 index 00000000000..09eec4fdee1 --- /dev/null +++ b/services/references/docker-compose.yml @@ -0,0 +1,40 @@ +# This file was auto-generated, do not edit it directly. +# Instead run bin/update_build_scripts from +# https://github.com/overleaf/internal/ + +services: + test_unit: + image: node:22.18.0 + volumes: + - .:/overleaf/services/references + - ../../node_modules:/overleaf/node_modules + - ../../libraries:/overleaf/libraries + working_dir: /overleaf/services/references + environment: + MOCHA_GREP: ${MOCHA_GREP} + LOG_LEVEL: ${LOG_LEVEL:-} + NODE_ENV: test + NODE_OPTIONS: "--unhandled-rejections=strict" + command: npm run --silent test:unit + user: node + + test_acceptance: + image: node:20.18.2 + volumes: + - .:/overleaf/services/references + - ../../node_modules:/overleaf/node_modules + - ../../libraries:/overleaf/libraries + working_dir: /overleaf/services/references + environment: + ELASTIC_SEARCH_DSN: es:9200 + MONGO_HOST: mongo + POSTGRES_HOST: postgres + MOCHA_GREP: ${MOCHA_GREP} + LOG_LEVEL: ${LOG_LEVEL:-} + NODE_ENV: test + NODE_OPTIONS: "--unhandled-rejections=strict" + user: node + depends_on: + mongo: + condition: service_started + command: npm run --silent test:acceptance diff --git a/services/references/package.json b/services/references/package.json new file mode 100644 index 00000000000..80480287b5e --- /dev/null +++ b/services/references/package.json @@ -0,0 +1,20 @@ +{ + "name": "@overleaf/references", + "description": "An API for providing citation-keys", + "private": true, + "type": "module", + "main": "app.js", + "scripts": { + "start": "node app.js" + }, + "version": "0.1.0", + "dependencies": { + "@overleaf/settings": "*", + "@overleaf/logger": "*", + "@overleaf/metrics": "*", + "async": "^3.2.5", + "express": "^4.22.1" + }, + "devDependencies": { + } +} diff --git a/services/references/tsconfig.json b/services/references/tsconfig.json new file mode 100644 index 00000000000..d3fdd3022ac --- /dev/null +++ b/services/references/tsconfig.json @@ -0,0 +1,12 @@ +{ + "extends": "../../tsconfig.backend.json", + "include": [ + "app.js", + "app/js/**/*", + "benchmarks/**/*", + "config/**/*", + "scripts/**/*", + "test/**/*", + "types" + ] +} diff --git a/services/web/config/settings.defaults.js b/services/web/config/settings.defaults.js index ee3025fa195..3b051d4f6ec 100644 --- a/services/web/config/settings.defaults.js +++ b/services/web/config/settings.defaults.js @@ -269,6 +269,9 @@ module.exports = { notifications: { url: `http://${process.env.NOTIFICATIONS_HOST || '127.0.0.1'}:3042`, }, + references: { + url: `http://${process.env.REFERENCES_HOST || '127.0.0.1'}:3056`, + }, webpack: { url: `http://${process.env.WEBPACK_HOST || '127.0.0.1'}:3808`, }, From 2d1d9461ef5b1689c29648e502c6f4cd8ab60747 Mon Sep 17 00:00:00 2001 From: Sam Van den Vonder Date: Wed, 4 Dec 2024 08:01:22 +0100 Subject: [PATCH 004/106] Enable Sandboxed Compiles feature --- server-ce/config/settings.js | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/server-ce/config/settings.js b/server-ce/config/settings.js index 7d9fd7702ee..e8bc5b17e2b 100644 --- a/server-ce/config/settings.js +++ b/server-ce/config/settings.js @@ -480,6 +480,41 @@ if ( ) } +// Overleaf Extended CE Compiler options to enable sandboxed compiles. +// ----------- +if (process.env.SANDBOXED_COMPILES === 'true') { + settings.clsi = { + ...settings.clsi, + dockerRunner: true, + docker: { + image: process.env.TEX_LIVE_DOCKER_IMAGE, + env: { + HOME: '/tmp', + PATH: + process.env.COMPILER_PATH || + '/usr/local/bin:/usr/bin:/bin', + }, + user: 'www-data', + } + } + + if (settings.path == null) { + settings.path = {} + } + settings.path.synctexBaseDir = () => '/compile' + if (process.env.SANDBOXED_COMPILES_SIBLING_CONTAINERS === 'true') { + console.log('Using sibling containers for sandboxed compiles') + if (process.env.SANDBOXED_COMPILES_HOST_DIR) { + settings.path.sandboxedCompilesHostDir = + process.env.SANDBOXED_COMPILES_HOST_DIR + } else { + console.error( + 'Sibling containers, but SANDBOXED_COMPILES_HOST_DIR not set' + ) + } + } +} + // With lots of incoming and outgoing HTTP connections to different services, // sometimes long running, it is a good idea to increase the default number // of sockets that Node will hold open. From c8a5246c34261dabf793158428f91309146d7dcf Mon Sep 17 00:00:00 2001 From: yu-i-i Date: Fri, 6 Dec 2024 12:45:15 +0100 Subject: [PATCH 005/106] Allow selecting a TeX Live image for a project --- server-ce/config/settings.js | 8 +- services/clsi/app/js/DockerRunner.js | 608 ++++++++++++++++++ .../Project/ProjectOptionsHandler.mjs | 1 - services/web/config/settings.defaults.js | 9 + 4 files changed, 618 insertions(+), 8 deletions(-) create mode 100644 services/clsi/app/js/DockerRunner.js diff --git a/server-ce/config/settings.js b/server-ce/config/settings.js index e8bc5b17e2b..dee89526615 100644 --- a/server-ce/config/settings.js +++ b/server-ce/config/settings.js @@ -488,13 +488,7 @@ if (process.env.SANDBOXED_COMPILES === 'true') { dockerRunner: true, docker: { image: process.env.TEX_LIVE_DOCKER_IMAGE, - env: { - HOME: '/tmp', - PATH: - process.env.COMPILER_PATH || - '/usr/local/bin:/usr/bin:/bin', - }, - user: 'www-data', + user: process.env.TEX_LIVE_DOCKER_USER || 'www-data', } } diff --git a/services/clsi/app/js/DockerRunner.js b/services/clsi/app/js/DockerRunner.js new file mode 100644 index 00000000000..ecfc6b5b546 --- /dev/null +++ b/services/clsi/app/js/DockerRunner.js @@ -0,0 +1,608 @@ +const { promisify } = require('node:util') +const Settings = require('@overleaf/settings') +const logger = require('@overleaf/logger') +const Docker = require('dockerode') +const dockerode = new Docker() +const crypto = require('node:crypto') +const async = require('async') +const LockManager = require('./DockerLockManager') +const Path = require('node:path') +const _ = require('lodash') + +const ONE_HOUR_IN_MS = 60 * 60 * 1000 +logger.debug('using docker runner') + +let containerMonitorTimeout +let containerMonitorInterval + +const DockerRunner = { + run( + projectId, + command, + directory, + image, + timeout, + environment, + compileGroup, + callback + ) { + command = command.map(arg => + arg.toString().replace('$COMPILE_DIR', '/compile') + ) + if (image == null) { + image = Settings.clsi.docker.image + } + + if ( + Settings.clsi.docker.allowedImages && + !Settings.clsi.docker.allowedImages.includes(image) + ) { + return callback(new Error('image not allowed')) + } + + if (Settings.texliveImageNameOveride != null) { + const img = image.split('/') + image = `${Settings.texliveImageNameOveride}/${img[2]}` + } + + if (compileGroup === 'synctex-output') { + // In: directory = '/overleaf/services/clsi/output/projectId-userId/generated-files/buildId' + // directory.split('/').slice(-3) === 'projectId-userId/generated-files/buildId' + // sandboxedCompilesHostDirOutput = '/host/output' + // Out: directory = '/host/output/projectId-userId/generated-files/buildId' + directory = Path.join( + Settings.path.sandboxedCompilesHostDirOutput, + ...directory.split('/').slice(-3) + ) + } else { + // In: directory = '/overleaf/services/clsi/compiles/projectId-userId' + // Path.basename(directory) === 'projectId-userId' + // sandboxedCompilesHostDirCompiles = '/host/compiles' + // Out: directory = '/host/compiles/projectId-userId' + directory = Path.join( + Settings.path.sandboxedCompilesHostDirCompiles, + Path.basename(directory) + ) + } + + const volumes = { [directory]: '/compile' } + if ( + compileGroup === 'synctex' || + compileGroup === 'synctex-output' || + compileGroup === 'wordcount' + ) { + volumes[directory] += ':ro' + } + + const options = DockerRunner._getContainerOptions( + command, + image, + volumes, + timeout, + environment, + compileGroup + ) + const fingerprint = DockerRunner._fingerprintContainer(options) + const name = `project-${projectId}-${fingerprint}` + options.name = name + + // logOptions = _.clone(options) + // logOptions?.HostConfig?.SecurityOpt = "secomp used, removed in logging" + logger.debug({ projectId }, 'running docker container') + DockerRunner._runAndWaitForContainer( + options, + volumes, + timeout, + (error, output) => { + if (error && error.statusCode === 500) { + logger.debug( + { err: error, projectId }, + 'error running container so destroying and retrying' + ) + DockerRunner.destroyContainer(name, null, true, error => { + if (error != null) { + return callback(error) + } + DockerRunner._runAndWaitForContainer( + options, + volumes, + timeout, + callback + ) + }) + } else { + callback(error, output) + } + } + ) + + // pass back the container name to allow it to be killed + return name + }, + + kill(containerId, callback) { + logger.debug({ containerId }, 'sending kill signal to container') + const container = dockerode.getContainer(containerId) + container.kill(error => { + if ( + error != null && + error.message != null && + error.message.match(/Cannot kill container .* is not running/) + ) { + logger.warn( + { err: error, containerId }, + 'container not running, continuing' + ) + error = null + } + if (error != null) { + logger.error({ err: error, containerId }, 'error killing container') + callback(error) + } else { + callback() + } + }) + }, + + _runAndWaitForContainer(options, volumes, timeout, _callback) { + const callback = _.once(_callback) + const { name } = options + + let streamEnded = false + let containerReturned = false + let output = {} + + function callbackIfFinished() { + if (streamEnded && containerReturned) { + callback(null, output) + } + } + + function attachStreamHandler(error, _output) { + if (error != null) { + return callback(error) + } + output = _output + streamEnded = true + callbackIfFinished() + } + + DockerRunner.startContainer( + options, + volumes, + attachStreamHandler, + (error, containerId) => { + if (error != null) { + return callback(error) + } + + DockerRunner.waitForContainer( + name, + timeout, + options, + (error, exitCode) => { + if (error != null) { + return callback(error) + } + if (exitCode === 137) { + // exit status from kill -9 + const err = new Error('terminated') + err.terminated = true + return callback(err) + } + if (exitCode === 1) { + // exit status from chktex + const err = new Error('exited') + err.code = exitCode + return callback(err) + } + containerReturned = true + logger.debug( + // The seccomp policy is very large. Avoid logging it. _.omit deep clones. + { exitCode, options: _.omit(options, 'HostConfig.SecurityOpt') }, + 'docker container has exited' + ) + callbackIfFinished() + } + ) + } + ) + }, + + _getContainerOptions( + command, + image, + volumes, + timeout, + environment, + compileGroup + ) { + const timeoutInSeconds = timeout / 1000 + + for (const hostVol in volumes) { + const dockerVol = volumes[hostVol] + if (volumes[hostVol].slice(-3).indexOf(':r') === -1) { + volumes[hostVol] = `${dockerVol}:rw` + } + } + + // merge settings and environment parameter + const env = {} + for (const src of [Settings.clsi.docker.env, environment || {}]) { + for (const key in src) { + const value = src[key] + env[key] = value + } + } + // set the path based on the image year + const match = image.match(/:([0-9]+)\.[0-9]+|:TL([0-9]+)/) + // the rolling build does not follow our .. convention + const year = match ? match[1] || match[2] : 'rolling' + env.PATH = `/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/texlive/${year}/bin/x86_64-linux/` + const options = { + Cmd: command, + Image: image, + WorkingDir: '/compile', + NetworkDisabled: true, + Memory: 1024 * 1024 * 1024 * 1024, // 1 Gb + User: Settings.clsi.docker.user, + Env: Object.entries(env).map(([key, value]) => `${key}=${value}`), + HostConfig: { + Binds: Object.entries(volumes).map( + ([hostVol, dockerVol]) => `${hostVol}:${dockerVol}` + ), + LogConfig: { Type: 'none', Config: {} }, + Ulimits: [ + { + Name: 'cpu', + Soft: timeoutInSeconds + 5, + Hard: timeoutInSeconds + 10, + }, + ], + CapDrop: ['ALL'], + SecurityOpt: ['no-new-privileges'], + }, + } + + if (Settings.clsi.docker.seccomp_profile != null) { + options.HostConfig.SecurityOpt.push( + `seccomp=${Settings.clsi.docker.seccomp_profile}` + ) + } + + if (Settings.clsi.docker.apparmor_profile != null) { + options.HostConfig.SecurityOpt.push( + `apparmor=${Settings.clsi.docker.apparmor_profile}` + ) + } + + if (Settings.clsi.docker.runtime) { + options.HostConfig.Runtime = Settings.clsi.docker.runtime + } + + if (Settings.clsi.docker.Readonly) { + options.HostConfig.ReadonlyRootfs = true + options.HostConfig.Tmpfs = { '/tmp': 'rw,noexec,nosuid,size=65536k' } + options.Volumes['/home/tex'] = {} + } + + // Allow per-compile group overriding of individual settings + if ( + Settings.clsi.docker.compileGroupConfig && + Settings.clsi.docker.compileGroupConfig[compileGroup] + ) { + const override = Settings.clsi.docker.compileGroupConfig[compileGroup] + for (const key in override) { + _.set(options, key, override[key]) + } + } + + return options + }, + + _fingerprintContainer(containerOptions) { + // Yay, Hashing! + const json = JSON.stringify(containerOptions) + return crypto.createHash('md5').update(json).digest('hex') + }, + + startContainer(options, volumes, attachStreamHandler, callback) { + LockManager.runWithLock( + options.name, + releaseLock => + DockerRunner._startContainer( + options, + volumes, + attachStreamHandler, + releaseLock + ), + callback + ) + }, + + // Check that volumes exist and are directories + _startContainer(options, volumes, attachStreamHandler, callback) { + callback = _.once(callback) + const { name } = options + + logger.debug({ containerName: name }, 'starting container') + const container = dockerode.getContainer(name) + + function createAndStartContainer() { + dockerode.createContainer(options, (error, container) => { + if (error != null) { + return callback(error) + } + startExistingContainer() + }) + } + + function startExistingContainer() { + DockerRunner.attachToContainer( + options.name, + attachStreamHandler, + error => { + if (error != null) { + return callback(error) + } + container.start(error => { + if (error != null && error.statusCode !== 304) { + callback(error) + } else { + // already running + callback() + } + }) + } + ) + } + + container.inspect((error, stats) => { + if (error != null && error.statusCode === 404) { + createAndStartContainer() + } else if (error != null) { + logger.err( + { containerName: name, error }, + 'unable to inspect container to start' + ) + callback(error) + } else { + startExistingContainer() + } + }) + }, + + attachToContainer(containerId, attachStreamHandler, attachStartCallback) { + const container = dockerode.getContainer(containerId) + container.attach({ stdout: 1, stderr: 1, stream: 1 }, (error, stream) => { + if (error != null) { + logger.error( + { err: error, containerId }, + 'error attaching to container' + ) + return attachStartCallback(error) + } else { + attachStartCallback() + } + + logger.debug({ containerId }, 'attached to container') + + const MAX_OUTPUT = 1024 * 1024 * 2 // limit output to 2MB + function createStringOutputStream(name) { + return { + data: '', + overflowed: false, + write(data) { + if (this.overflowed) { + return + } + if (this.data.length < MAX_OUTPUT) { + this.data += data + } else { + logger.info( + { + containerId, + length: this.data.length, + maxLen: MAX_OUTPUT, + }, + `${name} exceeds max size` + ) + this.data += `(...truncated at ${MAX_OUTPUT} chars...)` + this.overflowed = true + } + }, + // kill container if too much output + // docker.containers.kill(containerId, () ->) + } + } + + const stdout = createStringOutputStream('stdout') + const stderr = createStringOutputStream('stderr') + + container.modem.demuxStream(stream, stdout, stderr) + + stream.on('error', err => + logger.error( + { err, containerId }, + 'error reading from container stream' + ) + ) + + stream.on('end', () => + attachStreamHandler(null, { stdout: stdout.data, stderr: stderr.data }) + ) + }) + }, + + waitForContainer(containerId, timeout, options, _callback) { + const callback = _.once(_callback) + + const container = dockerode.getContainer(containerId) + + let timedOut = false + const timeoutId = setTimeout(() => { + timedOut = true + logger.debug({ containerId }, 'timeout reached, killing container') + container.kill(err => { + logger.warn({ err, containerId }, 'failed to kill container') + }) + }, timeout) + + logger.debug({ containerId }, 'waiting for docker container') + container.wait((error, res) => { + if (error?.statusCode === 404 && options.HostConfig.AutoRemove) { + logger.debug( + { containerId }, + 'auto-destroy container destroyed before starting to wait' + ) + clearTimeout(timeoutId) + return callback(null, 0) + } + if (error != null) { + clearTimeout(timeoutId) + logger.warn({ err: error, containerId }, 'error waiting for container') + return callback(error) + } + if (timedOut) { + logger.debug({ containerId }, 'docker container timed out') + error = new Error('container timed out') + error.timedout = true + callback(error) + } else { + clearTimeout(timeoutId) + logger.debug( + { containerId, exitCode: res.StatusCode }, + 'docker container returned' + ) + callback(null, res.StatusCode) + } + }) + }, + + destroyContainer(containerName, containerId, shouldForce, callback) { + // We want the containerName for the lock and, ideally, the + // containerId to delete. There is a bug in the docker.io module + // where if you delete by name and there is an error, it throws an + // async exception, but if you delete by id it just does a normal + // error callback. We fall back to deleting by name if no id is + // supplied. + LockManager.runWithLock( + containerName, + releaseLock => + DockerRunner._destroyContainer( + containerId || containerName, + shouldForce, + releaseLock + ), + callback + ) + }, + + _destroyContainer(containerId, shouldForce, callback) { + logger.debug({ containerId }, 'destroying docker container') + const container = dockerode.getContainer(containerId) + container.remove({ force: shouldForce === true, v: true }, error => { + if (error != null && error.statusCode === 404) { + logger.warn( + { err: error, containerId }, + 'container not found, continuing' + ) + error = null + } + if (error != null) { + logger.error({ err: error, containerId }, 'error destroying container') + } else { + logger.debug({ containerId }, 'destroyed container') + } + callback(error) + }) + }, + + // handle expiry of docker containers + + MAX_CONTAINER_AGE: Settings.clsi.docker.maxContainerAge || ONE_HOUR_IN_MS, + + examineOldContainer(container, callback) { + const name = container.Name || (container.Names && container.Names[0]) + const created = container.Created * 1000 // creation time is returned in seconds + const now = Date.now() + const age = now - created + const maxAge = DockerRunner.MAX_CONTAINER_AGE + const ttl = maxAge - age + logger.debug( + { containerName: name, created, now, age, maxAge, ttl }, + 'checking whether to destroy container' + ) + return { name, id: container.Id, ttl } + }, + + destroyOldContainers(callback) { + dockerode.listContainers({ all: true }, (error, containers) => { + if (error != null) { + return callback(error) + } + const jobs = [] + for (const container of containers) { + const { name, id, ttl } = DockerRunner.examineOldContainer(container) + if (name.slice(0, 9) === '/project-' && ttl <= 0) { + // strip the / prefix + // the LockManager uses the plain container name + const plainName = name.slice(1) + jobs.push(cb => + DockerRunner.destroyContainer(plainName, id, false, () => cb()) + ) + } + } + // Ignore errors because some containers get stuck but + // will be destroyed next time + async.series(jobs, callback) + }) + }, + + startContainerMonitor() { + logger.debug( + { maxAge: DockerRunner.MAX_CONTAINER_AGE }, + 'starting container expiry' + ) + + // guarantee only one monitor is running + DockerRunner.stopContainerMonitor() + + // randomise the start time + const randomDelay = Math.floor(Math.random() * 5 * 60 * 1000) + containerMonitorTimeout = setTimeout(() => { + containerMonitorInterval = setInterval( + () => + DockerRunner.destroyOldContainers(err => { + if (err) { + logger.error({ err }, 'failed to destroy old containers') + } + }), + ONE_HOUR_IN_MS + ) + }, randomDelay) + }, + + stopContainerMonitor() { + if (containerMonitorTimeout) { + clearTimeout(containerMonitorTimeout) + containerMonitorTimeout = undefined + } + if (containerMonitorInterval) { + clearInterval(containerMonitorInterval) + containerMonitorInterval = undefined + } + }, + + canRunSyncTeXInOutputDir() { + return Boolean(Settings.path.sandboxedCompilesHostDirOutput) + }, +} + +DockerRunner.startContainerMonitor() + +module.exports = DockerRunner +module.exports.promises = { + run: promisify(DockerRunner.run), + kill: promisify(DockerRunner.kill), +} diff --git a/services/web/app/src/Features/Project/ProjectOptionsHandler.mjs b/services/web/app/src/Features/Project/ProjectOptionsHandler.mjs index 8fcc0855281..972783093a7 100644 --- a/services/web/app/src/Features/Project/ProjectOptionsHandler.mjs +++ b/services/web/app/src/Features/Project/ProjectOptionsHandler.mjs @@ -40,7 +40,6 @@ const ProjectOptionsHandler = { if (!imageName || !Array.isArray(settings.allowedImageNames)) { return undefined } - imageName = imageName.toLowerCase() const isAllowed = settings.allowedImageNames.find( allowed => imageName === allowed.imageName ) diff --git a/services/web/config/settings.defaults.js b/services/web/config/settings.defaults.js index 3b051d4f6ec..381f5c4b09a 100644 --- a/services/web/config/settings.defaults.js +++ b/services/web/config/settings.defaults.js @@ -1097,6 +1097,15 @@ module.exports = { managedUsers: { enabled: false, }, + + allowedImageNames: process.env.SANDBOXED_COMPILES === 'true' + ? parseTextExtensions(process.env.ALL_TEX_LIVE_DOCKER_IMAGES) + .map((imageName, index) => ({ + imageName, + imageDesc: parseTextExtensions(process.env.ALL_TEX_LIVE_DOCKER_IMAGE_NAMES)[index] + || imageName.split(':')[1], + })) + : undefined, } module.exports.mergeWith = function (overrides) { From 275984fd852f46fc90271a5a9f8a75819aa0cade Mon Sep 17 00:00:00 2001 From: yu-i-i Date: Wed, 11 Dec 2024 03:55:41 +0100 Subject: [PATCH 006/106] Enable Symbol Palette --- services/web/config/settings.defaults.js | 3 +- .../components/symbol-palette-body.js | 61 ++ .../components/symbol-palette-close-button.js | 18 + .../components/symbol-palette-content.js | 94 ++ .../components/symbol-palette-info-link.js | 29 + .../components/symbol-palette-item.js | 67 ++ .../components/symbol-palette-items.js | 86 ++ .../components/symbol-palette-search.js | 44 + .../components/symbol-palette-tabs.js | 22 + .../components/symbol-palette.js | 8 + .../features/symbol-palette/data/symbols.json | 872 ++++++++++++++++++ .../symbol-palette/utils/categories.js | 44 + services/web/modules/symbol-palette/index.mjs | 2 + services/web/package.json | 1 + 14 files changed, 1350 insertions(+), 1 deletion(-) create mode 100644 services/web/frontend/js/features/symbol-palette/components/symbol-palette-body.js create mode 100644 services/web/frontend/js/features/symbol-palette/components/symbol-palette-close-button.js create mode 100644 services/web/frontend/js/features/symbol-palette/components/symbol-palette-content.js create mode 100644 services/web/frontend/js/features/symbol-palette/components/symbol-palette-info-link.js create mode 100644 services/web/frontend/js/features/symbol-palette/components/symbol-palette-item.js create mode 100644 services/web/frontend/js/features/symbol-palette/components/symbol-palette-items.js create mode 100644 services/web/frontend/js/features/symbol-palette/components/symbol-palette-search.js create mode 100644 services/web/frontend/js/features/symbol-palette/components/symbol-palette-tabs.js create mode 100644 services/web/frontend/js/features/symbol-palette/components/symbol-palette.js create mode 100644 services/web/frontend/js/features/symbol-palette/data/symbols.json create mode 100644 services/web/frontend/js/features/symbol-palette/utils/categories.js create mode 100644 services/web/modules/symbol-palette/index.mjs diff --git a/services/web/config/settings.defaults.js b/services/web/config/settings.defaults.js index 381f5c4b09a..8382fbf3a7a 100644 --- a/services/web/config/settings.defaults.js +++ b/services/web/config/settings.defaults.js @@ -1005,7 +1005,7 @@ module.exports = { pdfPreviewPromotions: [], diagnosticActions: [], sourceEditorCompletionSources: [], - sourceEditorSymbolPalette: [], + sourceEditorSymbolPalette: ['@/features/symbol-palette/components/symbol-palette'], sourceEditorToolbarComponents: [], sourceEditorToolbarEndButtons: [], rootContextProviders: [], @@ -1070,6 +1070,7 @@ module.exports = { 'launchpad', 'server-ce-scripts', 'user-activate', + 'symbol-palette', 'track-changes', ], viewIncludes: {}, diff --git a/services/web/frontend/js/features/symbol-palette/components/symbol-palette-body.js b/services/web/frontend/js/features/symbol-palette/components/symbol-palette-body.js new file mode 100644 index 00000000000..c4f47e325db --- /dev/null +++ b/services/web/frontend/js/features/symbol-palette/components/symbol-palette-body.js @@ -0,0 +1,61 @@ +import { TabPanels, TabPanel } from '@reach/tabs' +import { useTranslation } from 'react-i18next' +import PropTypes from 'prop-types' +import SymbolPaletteItems from './symbol-palette-items' + +export default function SymbolPaletteBody({ + categories, + categorisedSymbols, + filteredSymbols, + handleSelect, + focusInput, +}) { + const { t } = useTranslation() + + // searching with matches: show the matched symbols + // searching with no matches: show a message + // note: include empty tab panels so that aria-controls on tabs can still reference the panel ids + if (filteredSymbols) { + return ( + <> + {filteredSymbols.length ? ( + + ) : ( +
{t('no_symbols_found')}
+ )} + + + {categories.map(category => ( + + ))} + + + ) + } + + // not searching: show the symbols grouped by category + return ( + + {categories.map(category => ( + + + + ))} + + ) +} +SymbolPaletteBody.propTypes = { + categories: PropTypes.arrayOf(PropTypes.object).isRequired, + categorisedSymbols: PropTypes.object, + filteredSymbols: PropTypes.arrayOf(PropTypes.object), + handleSelect: PropTypes.func.isRequired, + focusInput: PropTypes.func.isRequired, +} diff --git a/services/web/frontend/js/features/symbol-palette/components/symbol-palette-close-button.js b/services/web/frontend/js/features/symbol-palette/components/symbol-palette-close-button.js new file mode 100644 index 00000000000..c472c315868 --- /dev/null +++ b/services/web/frontend/js/features/symbol-palette/components/symbol-palette-close-button.js @@ -0,0 +1,18 @@ +import { Button } from 'react-bootstrap' +import { useEditorContext } from '../../../shared/context/editor-context' + +export default function SymbolPaletteCloseButton() { + const { toggleSymbolPalette } = useEditorContext() + + return ( + + ) +} + diff --git a/services/web/frontend/js/features/symbol-palette/components/symbol-palette-content.js b/services/web/frontend/js/features/symbol-palette/components/symbol-palette-content.js new file mode 100644 index 00000000000..8537e145851 --- /dev/null +++ b/services/web/frontend/js/features/symbol-palette/components/symbol-palette-content.js @@ -0,0 +1,94 @@ +import { Tabs } from '@reach/tabs' +import { useCallback, useEffect, useMemo, useRef, useState } from 'react' +import { useTranslation } from 'react-i18next' +import PropTypes from 'prop-types' +import { matchSorter } from 'match-sorter' + +import symbols from '../data/symbols.json' +import { buildCategorisedSymbols, createCategories } from '../utils/categories' +import SymbolPaletteSearch from './symbol-palette-search' +import SymbolPaletteBody from './symbol-palette-body' +import SymbolPaletteTabs from './symbol-palette-tabs' +// import SymbolPaletteInfoLink from './symbol-palette-info-link' +import SymbolPaletteCloseButton from './symbol-palette-close-button' + +import '@reach/tabs/styles.css' + +export default function SymbolPaletteContent({ handleSelect }) { + const [input, setInput] = useState('') + + const { t } = useTranslation() + + // build the list of categories with translated labels + const categories = useMemo(() => createCategories(t), [t]) + + // group the symbols by category + const categorisedSymbols = useMemo( + () => buildCategorisedSymbols(categories), + [categories] + ) + + // select symbols which match the input + const filteredSymbols = useMemo(() => { + if (input === '') { + return null + } + + const words = input.trim().split(/\s+/) + + return words.reduceRight( + (symbols, word) => + matchSorter(symbols, word, { + keys: ['command', 'description', 'character', 'aliases'], + threshold: matchSorter.rankings.CONTAINS, + }), + symbols + ) + }, [input]) + + const inputRef = useRef(null) + + // allow the input to be focused + const focusInput = useCallback(() => { + if (inputRef.current) { + inputRef.current.focus() + } + }, []) + + // focus the input when the symbol palette is opened + useEffect(() => { + if (inputRef.current) { + inputRef.current.focus() + } + }, []) + + return ( + +
+
+
+ +
+ {/* Useless button (uncomment if you see any sense in it) */} + {/* */} + +
+
+ +
+
+ +
+
+ + ) +} +SymbolPaletteContent.propTypes = { + handleSelect: PropTypes.func.isRequired, +} diff --git a/services/web/frontend/js/features/symbol-palette/components/symbol-palette-info-link.js b/services/web/frontend/js/features/symbol-palette/components/symbol-palette-info-link.js new file mode 100644 index 00000000000..ba56cf2b106 --- /dev/null +++ b/services/web/frontend/js/features/symbol-palette/components/symbol-palette-info-link.js @@ -0,0 +1,29 @@ +import { Button, OverlayTrigger, Tooltip } from 'react-bootstrap' +import { useTranslation } from 'react-i18next' + +export default function SymbolPaletteInfoLink() { + const { t } = useTranslation() + + return ( + + {t('find_out_more_about_latex_symbols')} + + } + > + + + ) +} diff --git a/services/web/frontend/js/features/symbol-palette/components/symbol-palette-item.js b/services/web/frontend/js/features/symbol-palette/components/symbol-palette-item.js new file mode 100644 index 00000000000..a892f33cf8b --- /dev/null +++ b/services/web/frontend/js/features/symbol-palette/components/symbol-palette-item.js @@ -0,0 +1,67 @@ +import { useEffect, useRef } from 'react' +import { OverlayTrigger, Tooltip } from 'react-bootstrap' +import PropTypes from 'prop-types' + +export default function SymbolPaletteItem({ + focused, + handleSelect, + handleKeyDown, + symbol, +}) { + const buttonRef = useRef(null) + + // call focus() on this item when appropriate + useEffect(() => { + if ( + focused && + buttonRef.current && + document.activeElement?.closest('.symbol-palette-items') + ) { + buttonRef.current.focus() + } + }, [focused]) + + return ( + +
+ {symbol.description} +
+
{symbol.command}
+ {symbol.notes && ( +
{symbol.notes}
+ )} + + } + > + + + ) +} +SymbolPaletteItem.propTypes = { + symbol: PropTypes.shape({ + codepoint: PropTypes.string.isRequired, + description: PropTypes.string.isRequired, + command: PropTypes.string.isRequired, + character: PropTypes.string.isRequired, + notes: PropTypes.string, + }), + handleKeyDown: PropTypes.func.isRequired, + handleSelect: PropTypes.func.isRequired, + focused: PropTypes.bool, +} diff --git a/services/web/frontend/js/features/symbol-palette/components/symbol-palette-items.js b/services/web/frontend/js/features/symbol-palette/components/symbol-palette-items.js new file mode 100644 index 00000000000..44835261f5c --- /dev/null +++ b/services/web/frontend/js/features/symbol-palette/components/symbol-palette-items.js @@ -0,0 +1,86 @@ +import { useCallback, useEffect, useState } from 'react' +import PropTypes from 'prop-types' +import SymbolPaletteItem from './symbol-palette-item' + +export default function SymbolPaletteItems({ + items, + handleSelect, + focusInput, +}) { + const [focusedIndex, setFocusedIndex] = useState(0) + + // reset the focused item when the list of items changes + useEffect(() => { + setFocusedIndex(0) + }, [items]) + + // navigate through items with left and right arrows + const handleKeyDown = useCallback( + event => { + if (event.metaKey || event.altKey || event.ctrlKey || event.shiftKey) { + return + } + + switch (event.key) { + // focus previous item + case 'ArrowLeft': + case 'ArrowUp': + setFocusedIndex(index => (index > 0 ? index - 1 : items.length - 1)) + break + + // focus next item + case 'ArrowRight': + case 'ArrowDown': + setFocusedIndex(index => (index < items.length - 1 ? index + 1 : 0)) + break + + // focus first item + case 'Home': + setFocusedIndex(0) + break + + // focus last item + case 'End': + setFocusedIndex(items.length - 1) + break + + // allow the default action + case 'Enter': + case ' ': + break + + // any other key returns focus to the input + default: + focusInput() + break + } + }, + [focusInput, items.length] + ) + + return ( +
+ {items.map((symbol, index) => ( + { + handleSelect(symbol) + setFocusedIndex(index) + }} + handleKeyDown={handleKeyDown} + focused={index === focusedIndex} + /> + ))} +
+ ) +} +SymbolPaletteItems.propTypes = { + items: PropTypes.arrayOf( + PropTypes.shape({ + codepoint: PropTypes.string.isRequired, + }) + ).isRequired, + handleSelect: PropTypes.func.isRequired, + focusInput: PropTypes.func.isRequired, +} diff --git a/services/web/frontend/js/features/symbol-palette/components/symbol-palette-search.js b/services/web/frontend/js/features/symbol-palette/components/symbol-palette-search.js new file mode 100644 index 00000000000..cf5a1eb2a75 --- /dev/null +++ b/services/web/frontend/js/features/symbol-palette/components/symbol-palette-search.js @@ -0,0 +1,44 @@ +import { useCallback, useEffect, useState } from 'react' +import { useTranslation } from 'react-i18next' +import PropTypes from 'prop-types' +import { FormControl } from 'react-bootstrap' +import useDebounce from '../../../shared/hooks/use-debounce' + +export default function SymbolPaletteSearch({ setInput, inputRef }) { + const [localInput, setLocalInput] = useState('') + + // debounce the search input until a typing delay + const debouncedLocalInput = useDebounce(localInput, 250) + + useEffect(() => { + setInput(debouncedLocalInput) + }, [debouncedLocalInput, setInput]) + + const { t } = useTranslation() + + const inputRefCallback = useCallback( + element => { + inputRef.current = element + }, + [inputRef] + ) + + return ( + { + setLocalInput(event.target.value) + }} + /> + ) +} +SymbolPaletteSearch.propTypes = { + setInput: PropTypes.func.isRequired, + inputRef: PropTypes.object.isRequired, +} diff --git a/services/web/frontend/js/features/symbol-palette/components/symbol-palette-tabs.js b/services/web/frontend/js/features/symbol-palette/components/symbol-palette-tabs.js new file mode 100644 index 00000000000..d53cd93ac0e --- /dev/null +++ b/services/web/frontend/js/features/symbol-palette/components/symbol-palette-tabs.js @@ -0,0 +1,22 @@ +import { TabList, Tab } from '@reach/tabs' +import PropTypes from 'prop-types' + +export default function SymbolPaletteTabs({ categories }) { + return ( + + {categories.map(category => ( + + {category.label} + + ))} + + ) +} +SymbolPaletteTabs.propTypes = { + categories: PropTypes.arrayOf( + PropTypes.shape({ + id: PropTypes.string.isRequired, + label: PropTypes.string.isRequired, + }) + ).isRequired, +} diff --git a/services/web/frontend/js/features/symbol-palette/components/symbol-palette.js b/services/web/frontend/js/features/symbol-palette/components/symbol-palette.js new file mode 100644 index 00000000000..2f1cc5e8c88 --- /dev/null +++ b/services/web/frontend/js/features/symbol-palette/components/symbol-palette.js @@ -0,0 +1,8 @@ +import SymbolPaletteContent from './symbol-palette-content' + +export default function SymbolPalette() { + const handleSelect = (symbol) => { + window.dispatchEvent(new CustomEvent('editor:insert-symbol', { detail: symbol })) + } + return +} diff --git a/services/web/frontend/js/features/symbol-palette/data/symbols.json b/services/web/frontend/js/features/symbol-palette/data/symbols.json new file mode 100644 index 00000000000..af160b3eedf --- /dev/null +++ b/services/web/frontend/js/features/symbol-palette/data/symbols.json @@ -0,0 +1,872 @@ +[ + { + "category": "Greek", + "command": "\\alpha", + "codepoint": "U+1D6FC", + "description": "Lowercase Greek letter alpha", + "aliases": ["a", "α"], + "notes": "" + }, + { + "category": "Greek", + "command": "\\beta", + "codepoint": "U+1D6FD", + "description": "Lowercase Greek letter beta", + "aliases": ["b", "β"], + "notes": "" + }, + { + "category": "Greek", + "command": "\\gamma", + "codepoint": "U+1D6FE", + "description": "Lowercase Greek letter gamma", + "aliases": ["γ"], + "notes": "" + }, + { + "category": "Greek", + "command": "\\delta", + "codepoint": "U+1D6FF", + "description": "Lowercase Greek letter delta", + "aliases": ["δ"], + "notes": "" + }, + { + "category": "Greek", + "command": "\\varepsilon", + "codepoint": "U+1D700", + "description": "Lowercase Greek letter epsilon, varepsilon", + "aliases": ["ε"], + "notes": "" + }, + { + "category": "Greek", + "command": "\\epsilon", + "codepoint": "U+1D716", + "description": "Lowercase Greek letter epsilon lunate", + "aliases": ["ε"], + "notes": "" + }, + { + "category": "Greek", + "command": "\\zeta", + "codepoint": "U+1D701", + "description": "Lowercase Greek letter zeta", + "aliases": ["ζ"], + "notes": "" + }, + { + "category": "Greek", + "command": "\\eta", + "codepoint": "U+1D702", + "description": "Lowercase Greek letter eta", + "aliases": ["η"], + "notes": "" + }, + { + "category": "Greek", + "command": "\\vartheta", + "codepoint": "U+1D717", + "description": "Lowercase Greek letter curly theta, vartheta", + "aliases": ["θ"], + "notes": "" + }, + { + "category": "Greek", + "command": "\\theta", + "codepoint": "U+1D703", + "description": "Lowercase Greek letter theta", + "aliases": ["θ"], + "notes": "" + }, + { + "category": "Greek", + "command": "\\iota", + "codepoint": "U+1D704", + "description": "Lowercase Greek letter iota", + "aliases": ["ι"], + "notes": "" + }, + { + "category": "Greek", + "command": "\\kappa", + "codepoint": "U+1D705", + "description": "Lowercase Greek letter kappa", + "aliases": ["κ"], + "notes": "" + }, + { + "category": "Greek", + "command": "\\lambda", + "codepoint": "U+1D706", + "description": "Lowercase Greek letter lambda", + "aliases": ["λ"], + "notes": "" + }, + { + "category": "Greek", + "command": "\\mu", + "codepoint": "U+1D707", + "description": "Lowercase Greek letter mu", + "aliases": ["μ"], + "notes": "" + }, + { + "category": "Greek", + "command": "\\nu", + "codepoint": "U+1D708", + "description": "Lowercase Greek letter nu", + "aliases": ["ν"], + "notes": "" + }, + { + "category": "Greek", + "command": "\\xi", + "codepoint": "U+1D709", + "description": "Lowercase Greek letter xi", + "aliases": ["ξ"], + "notes": "" + }, + { + "category": "Greek", + "command": "\\pi", + "codepoint": "U+1D70B", + "description": "Lowercase Greek letter pi", + "aliases": ["π"], + "notes": "" + }, + { + "category": "Greek", + "command": "\\varrho", + "codepoint": "U+1D71A", + "description": "Lowercase Greek letter rho, varrho", + "aliases": ["ρ"], + "notes": "" + }, + { + "category": "Greek", + "command": "\\rho", + "codepoint": "U+1D70C", + "description": "Lowercase Greek letter rho", + "aliases": ["ρ"], + "notes": "" + }, + { + "category": "Greek", + "command": "\\sigma", + "codepoint": "U+1D70E", + "description": "Lowercase Greek letter sigma", + "aliases": ["σ"], + "notes": "" + }, + { + "category": "Greek", + "command": "\\varsigma", + "codepoint": "U+1D70D", + "description": "Lowercase Greek letter final sigma, varsigma", + "aliases": ["ς"], + "notes": "" + }, + { + "category": "Greek", + "command": "\\tau", + "codepoint": "U+1D70F", + "description": "Lowercase Greek letter tau", + "aliases": ["τ"], + "notes": "" + }, + { + "category": "Greek", + "command": "\\upsilon", + "codepoint": "U+1D710", + "description": "Lowercase Greek letter upsilon", + "aliases": ["υ"], + "notes": "" + }, + { + "category": "Greek", + "command": "\\phi", + "codepoint": "U+1D719", + "description": "Lowercase Greek letter phi", + "aliases": ["φ"], + "notes": "" + }, + { + "category": "Greek", + "command": "\\varphi", + "codepoint": "U+1D711", + "description": "Lowercase Greek letter phi, varphi", + "aliases": ["φ"], + "notes": "" + }, + { + "category": "Greek", + "command": "\\chi", + "codepoint": "U+1D712", + "description": "Lowercase Greek letter chi", + "aliases": ["χ"], + "notes": "" + }, + { + "category": "Greek", + "command": "\\psi", + "codepoint": "U+1D713", + "description": "Lowercase Greek letter psi", + "aliases": ["ψ"], + "notes": "" + }, + { + "category": "Greek", + "command": "\\omega", + "codepoint": "U+1D714", + "description": "Lowercase Greek letter omega", + "aliases": ["ω"], + "notes": "" + }, + { + "category": "Greek", + "command": "\\Gamma", + "codepoint": "U+00393", + "description": "Uppercase Greek letter Gamma", + "aliases": ["Γ"], + "notes": "" + }, + { + "category": "Greek", + "command": "\\Delta", + "codepoint": "U+00394", + "description": "Uppercase Greek letter Delta", + "aliases": ["Δ"], + "notes": "" + }, + { + "category": "Greek", + "command": "\\Theta", + "codepoint": "U+00398", + "description": "Uppercase Greek letter Theta", + "aliases": ["Θ"], + "notes": "" + }, + { + "category": "Greek", + "command": "\\Lambda", + "codepoint": "U+0039B", + "description": "Uppercase Greek letter Lambda", + "aliases": ["Λ"], + "notes": "" + }, + { + "category": "Greek", + "command": "\\Xi", + "codepoint": "U+0039E", + "description": "Uppercase Greek letter Xi", + "aliases": ["Ξ"], + "notes": "" + }, + { + "category": "Greek", + "command": "\\Pi", + "codepoint": "U+003A0", + "description": "Uppercase Greek letter Pi", + "aliases": ["Π"], + "notes": "Use \\prod for the product." + }, + { + "category": "Greek", + "command": "\\Sigma", + "codepoint": "U+003A3", + "description": "Uppercase Greek letter Sigma", + "aliases": ["Σ"], + "notes": "Use \\sum for the sum." + }, + { + "category": "Greek", + "command": "\\Upsilon", + "codepoint": "U+003A5", + "description": "Uppercase Greek letter Upsilon", + "aliases": ["Υ"], + "notes": "" + }, + { + "category": "Greek", + "command": "\\Phi", + "codepoint": "U+003A6", + "description": "Uppercase Greek letter Phi", + "aliases": ["Φ"], + "notes": "" + }, + { + "category": "Greek", + "command": "\\Psi", + "codepoint": "U+003A8", + "description": "Uppercase Greek letter Psi", + "aliases": ["Ψ"], + "notes": "" + }, + { + "category": "Greek", + "command": "\\Omega", + "codepoint": "U+003A9", + "description": "Uppercase Greek letter Omega", + "aliases": ["Ω"], + "notes": "" + }, + { + "category": "Relations", + "command": "\\neq", + "codepoint": "U+02260", + "description": "Not equal", + "aliases": ["!="], + "notes": "" + }, + { + "category": "Relations", + "command": "\\leq", + "codepoint": "U+02264", + "description": "Less than or equal", + "aliases": ["<="], + "notes": "" + }, + { + "category": "Relations", + "command": "\\geq", + "codepoint": "U+02265", + "description": "Greater than or equal", + "aliases": [">="], + "notes": "" + }, + { + "category": "Relations", + "command": "\\ll", + "codepoint": "U+0226A", + "description": "Much less than", + "aliases": ["<<"], + "notes": "" + }, + { + "category": "Relations", + "command": "\\gg", + "codepoint": "U+0226B", + "description": "Much greater than", + "aliases": [">>"], + "notes": "" + }, + { + "category": "Relations", + "command": "\\prec", + "codepoint": "U+0227A", + "description": "Precedes", + "notes": "" + }, + { + "category": "Relations", + "command": "\\succ", + "codepoint": "U+0227B", + "description": "Succeeds", + "notes": "" + }, + { + "category": "Relations", + "command": "\\in", + "codepoint": "U+02208", + "description": "Set membership", + "notes": "" + }, + { + "category": "Relations", + "command": "\\notin", + "codepoint": "U+02209", + "description": "Negated set membership", + "notes": "" + }, + { + "category": "Relations", + "command": "\\ni", + "codepoint": "U+0220B", + "description": "Contains", + "notes": "" + }, + { + "category": "Relations", + "command": "\\subset", + "codepoint": "U+02282", + "description": "Subset", + "notes": "" + }, + { + "category": "Relations", + "command": "\\subseteq", + "codepoint": "U+02286", + "description": "Subset or equals", + "notes": "" + }, + { + "category": "Relations", + "command": "\\supset", + "codepoint": "U+02283", + "description": "Superset", + "notes": "" + }, + { + "category": "Relations", + "command": "\\simeq", + "codepoint": "U+02243", + "description": "Similar", + "notes": "" + }, + { + "category": "Relations", + "command": "\\approx", + "codepoint": "U+02248", + "description": "Approximate", + "notes": "" + }, + { + "category": "Relations", + "command": "\\equiv", + "codepoint": "U+02261", + "description": "Identical with", + "notes": "" + }, + { + "category": "Relations", + "command": "\\cong", + "codepoint": "U+02245", + "description": "Congruent with", + "notes": "" + }, + { + "category": "Relations", + "command": "\\mid", + "codepoint": "U+02223", + "description": "Mid, divides, vertical bar, modulus, absolute value", + "notes": "Use \\lvert...\\rvert for the absolute value." + }, + { + "category": "Relations", + "command": "\\nmid", + "codepoint": "U+02224", + "description": "Negated mid, not divides", + "notes": "Requires \\usepackage{amssymb}." + }, + { + "category": "Relations", + "command": "\\parallel", + "codepoint": "U+02225", + "description": "Parallel, double vertical bar, norm", + "notes": "Use \\lVert...\\rVert for the norm." + }, + { + "category": "Relations", + "command": "\\perp", + "codepoint": "U+027C2", + "description": "Perpendicular", + "notes": "" + }, + { + "category": "Operators", + "command": "\\times", + "codepoint": "U+000D7", + "description": "Cross product, multiplication", + "aliases": ["x"], + "notes": "" + }, + { + "category": "Operators", + "command": "\\div", + "codepoint": "U+000F7", + "description": "Division", + "notes": "" + }, + { + "category": "Operators", + "command": "\\cap", + "codepoint": "U+02229", + "description": "Intersection", + "notes": "" + }, + { + "category": "Operators", + "command": "\\cup", + "codepoint": "U+0222A", + "description": "Union", + "notes": "" + }, + { + "category": "Operators", + "command": "\\cdot", + "codepoint": "U+022C5", + "description": "Dot product, multiplication", + "notes": "" + }, + { + "category": "Operators", + "command": "\\cdots", + "codepoint": "U+022EF", + "description": "Centered dots", + "notes": "" + }, + { + "category": "Operators", + "command": "\\bullet", + "codepoint": "U+02219", + "description": "Bullet", + "notes": "" + }, + { + "category": "Operators", + "command": "\\circ", + "codepoint": "U+025E6", + "description": "Circle", + "notes": "" + }, + { + "category": "Operators", + "command": "\\wedge", + "codepoint": "U+02227", + "description": "Wedge, logical and", + "notes": "" + }, + { + "category": "Operators", + "command": "\\vee", + "codepoint": "U+02228", + "description": "Vee, logical or", + "notes": "" + }, + { + "category": "Operators", + "command": "\\setminus", + "codepoint": "U+0005C", + "description": "Set minus, backslash", + "notes": "Use \\backslash for a backslash." + }, + { + "category": "Operators", + "command": "\\oplus", + "codepoint": "U+02295", + "description": "Plus sign in circle", + "notes": "" + }, + { + "category": "Operators", + "command": "\\otimes", + "codepoint": "U+02297", + "description": "Multiply sign in circle", + "notes": "" + }, + { + "category": "Operators", + "command": "\\sum", + "codepoint": "U+02211", + "description": "Summation operator", + "notes": "Use \\Sigma for the letter Sigma." + }, + { + "category": "Operators", + "command": "\\prod", + "codepoint": "U+0220F", + "description": "Product operator", + "notes": "Use \\Pi for the letter Pi." + }, + { + "category": "Operators", + "command": "\\bigcap", + "codepoint": "U+022C2", + "description": "Intersection operator", + "notes": "" + }, + { + "category": "Operators", + "command": "\\bigcup", + "codepoint": "U+022C3", + "description": "Union operator", + "notes": "" + }, + { + "category": "Operators", + "command": "\\int", + "codepoint": "U+0222B", + "description": "Integral operator", + "notes": "" + }, + { + "category": "Operators", + "command": "\\iint", + "codepoint": "U+0222C", + "description": "Double integral operator", + "notes": "Requires \\usepackage{amsmath}." + }, + { + "category": "Operators", + "command": "\\iiint", + "codepoint": "U+0222D", + "description": "Triple integral operator", + "notes": "Requires \\usepackage{amsmath}." + }, + { + "category": "Arrows", + "command": "\\leftarrow", + "codepoint": "U+02190", + "description": "Leftward arrow", + "aliases": ["<-"], + "notes": "" + }, + { + "category": "Arrows", + "command": "\\rightarrow", + "codepoint": "U+02192", + "description": "Rightward arrow", + "aliases": ["->"], + "notes": "" + }, + { + "category": "Arrows", + "command": "\\leftrightarrow", + "codepoint": "U+02194", + "description": "Left and right arrow", + "aliases": ["<->"], + "notes": "" + }, + { + "category": "Arrows", + "command": "\\uparrow", + "codepoint": "U+02191", + "description": "Upward arrow", + "notes": "" + }, + { + "category": "Arrows", + "command": "\\downarrow", + "codepoint": "U+02193", + "description": "Downward arrow", + "notes": "" + }, + { + "category": "Arrows", + "command": "\\Leftarrow", + "codepoint": "U+021D0", + "description": "Is implied by", + "aliases": ["<="], + "notes": "" + }, + { + "category": "Arrows", + "command": "\\Rightarrow", + "codepoint": "U+021D2", + "description": "Implies", + "aliases": ["=>"], + "notes": "" + }, + { + "category": "Arrows", + "command": "\\Leftrightarrow", + "codepoint": "U+021D4", + "description": "Left and right double arrow", + "aliases": ["<=>"], + "notes": "" + }, + { + "category": "Arrows", + "command": "\\mapsto", + "codepoint": "U+021A6", + "description": "Maps to, rightward", + "notes": "" + }, + { + "category": "Arrows", + "command": "\\nearrow", + "codepoint": "U+02197", + "description": "NE pointing arrow", + "notes": "" + }, + { + "category": "Arrows", + "command": "\\searrow", + "codepoint": "U+02198", + "description": "SE pointing arrow", + "notes": "" + }, + { + "category": "Arrows", + "command": "\\rightleftharpoons", + "codepoint": "U+021CC", + "description": "Right harpoon over left", + "notes": "" + }, + { + "category": "Arrows", + "command": "\\leftharpoonup", + "codepoint": "U+021BC", + "description": "Left harpoon up", + "notes": "" + }, + { + "category": "Arrows", + "command": "\\rightharpoonup", + "codepoint": "U+021C0", + "description": "Right harpoon up", + "notes": "" + }, + { + "category": "Arrows", + "command": "\\leftharpoondown", + "codepoint": "U+021BD", + "description": "Left harpoon down", + "notes": "" + }, + { + "category": "Arrows", + "command": "\\rightharpoondown", + "codepoint": "U+021C1", + "description": "Right harpoon down", + "notes": "" + }, + { + "category": "Misc", + "command": "\\infty", + "codepoint": "U+0221E", + "description": "Infinity", + "notes": "" + }, + { + "category": "Misc", + "command": "\\partial", + "codepoint": "U+1D715", + "description": "Partial differential", + "notes": "" + }, + { + "category": "Misc", + "command": "\\nabla", + "codepoint": "U+02207", + "description": "Nabla, del, hamilton operator", + "notes": "" + }, + { + "category": "Misc", + "command": "\\varnothing", + "codepoint": "U+02300", + "description": "Empty set", + "notes": "Requires \\usepackage{amssymb}." + }, + { + "category": "Misc", + "command": "\\forall", + "codepoint": "U+02200", + "description": "For all", + "notes": "" + }, + { + "category": "Misc", + "command": "\\exists", + "codepoint": "U+02203", + "description": "There exists", + "notes": "" + }, + { + "category": "Misc", + "command": "\\neg", + "codepoint": "U+000AC", + "description": "Not sign", + "notes": "" + }, + { + "category": "Misc", + "command": "\\Re", + "codepoint": "U+0211C", + "description": "Real part", + "notes": "" + }, + { + "category": "Misc", + "command": "\\Im", + "codepoint": "U+02111", + "description": "Imaginary part", + "notes": "" + }, + { + "category": "Misc", + "command": "\\Box", + "codepoint": "U+025A1", + "description": "Square", + "notes": "Requires \\usepackage{amssymb}." + }, + { + "category": "Misc", + "command": "\\triangle", + "codepoint": "U+025B3", + "description": "Triangle", + "notes": "" + }, + { + "category": "Misc", + "command": "\\aleph", + "codepoint": "U+02135", + "description": "Hebrew letter aleph", + "notes": "" + }, + { + "category": "Misc", + "command": "\\wp", + "codepoint": "U+02118", + "description": "Weierstrass letter p", + "notes": "" + }, + { + "category": "Misc", + "command": "\\#", + "codepoint": "U+00023", + "description": "Number sign, hashtag", + "notes": "" + }, + { + "category": "Misc", + "command": "\\$", + "codepoint": "U+00024", + "description": "Dollar sign", + "notes": "" + }, + { + "category": "Misc", + "command": "\\%", + "codepoint": "U+00025", + "description": "Percent sign", + "notes": "" + }, + { + "category": "Misc", + "command": "\\&", + "codepoint": "U+00026", + "description": "Et sign, and, ampersand", + "notes": "" + }, + { + "category": "Misc", + "command": "\\{", + "codepoint": "U+0007B", + "description": "Left curly brace", + "notes": "" + }, + { + "category": "Misc", + "command": "\\}", + "codepoint": "U+0007D", + "description": "Right curly brace", + "notes": "" + }, + { + "category": "Misc", + "command": "\\langle", + "codepoint": "U+027E8", + "description": "Left angle bracket, bra", + "notes": "" + }, + { + "category": "Misc", + "command": "\\rangle", + "codepoint": "U+027E9", + "description": "Right angle bracket, ket", + "notes": "" + } +] diff --git a/services/web/frontend/js/features/symbol-palette/utils/categories.js b/services/web/frontend/js/features/symbol-palette/utils/categories.js new file mode 100644 index 00000000000..872534771fe --- /dev/null +++ b/services/web/frontend/js/features/symbol-palette/utils/categories.js @@ -0,0 +1,44 @@ +import symbols from '../data/symbols.json' +export function createCategories(t) { + return [ + { + id: 'Greek', + label: t('category_greek'), + }, + { + id: 'Arrows', + label: t('category_arrows'), + }, + { + id: 'Operators', + label: t('category_operators'), + }, + { + id: 'Relations', + label: t('category_relations'), + }, + { + id: 'Misc', + label: t('category_misc'), + }, + ] +} + +export function buildCategorisedSymbols(categories) { + const output = {} + + for (const category of categories) { + output[category.id] = [] + } + + for (const item of symbols) { + if (item.category in output) { + item.character = String.fromCodePoint( + parseInt(item.codepoint.replace(/^U\+0*/, ''), 16) + ) + output[item.category].push(item) + } + } + + return output +} diff --git a/services/web/modules/symbol-palette/index.mjs b/services/web/modules/symbol-palette/index.mjs new file mode 100644 index 00000000000..3a412c2eec6 --- /dev/null +++ b/services/web/modules/symbol-palette/index.mjs @@ -0,0 +1,2 @@ +import logger from '@overleaf/logger' +logger.debug({}, 'Enable Symbol Palette') diff --git a/services/web/package.json b/services/web/package.json index 88c0d05df1a..d9bb858e523 100644 --- a/services/web/package.json +++ b/services/web/package.json @@ -228,6 +228,7 @@ "@pollyjs/adapter-node-http": "^6.0.6", "@pollyjs/core": "^6.0.6", "@pollyjs/persister-fs": "^6.0.6", + "@reach/tabs": "0.18.0", "@replit/codemirror-emacs": "overleaf/codemirror-emacs#4394c03858f27053f8768258e9493866e06e938e", "@replit/codemirror-indentation-markers": "overleaf/codemirror-indentation-markers#371ce3b56f453a392eb0d3b85ab019c185c68e1f", "@replit/codemirror-vim": "overleaf/codemirror-vim#1bef138382d948018f3f9b8a4d7a70ab61774e4b", From 01f802e8e933376756942f5d1ef1565bb0ada066 Mon Sep 17 00:00:00 2001 From: yu-i-i Date: Tue, 3 Dec 2024 01:18:19 +0100 Subject: [PATCH 007/106] Enable LDAP and SAML authentication support --- patches/@node-saml+node-saml+4.0.5.patch | 23 +++ patches/ldapauth-fork+4.3.3.patch | 64 +++++++ .../AuthenticationController.mjs | 5 +- .../PasswordReset/PasswordResetController.mjs | 4 + .../PasswordReset/PasswordResetHandler.mjs | 4 + .../app/src/Features/User/UserController.mjs | 3 +- services/web/app/views/user/login.pug | 12 +- services/web/app/views/user/settings.pug | 4 +- services/web/config/settings.defaults.js | 2 + services/web/locales/en.json | 2 + .../launchpad/app/src/LaunchpadController.mjs | 3 +- .../modules/launchpad/app/views/launchpad.pug | 70 ++++++++ .../app/src/AuthenticationControllerLdap.mjs | 64 +++++++ .../app/src/AuthenticationManagerLdap.mjs | 80 +++++++++ .../app/src/InitLdapSettings.mjs | 17 ++ .../app/src/LdapContacts.mjs | 136 +++++++++++++++ .../app/src/LdapStrategy.mjs | 78 +++++++++ .../web/modules/ldap-authentication/index.mjs | 30 ++++ .../app/src/AuthenticationControllerSaml.mjs | 160 ++++++++++++++++++ .../app/src/AuthenticationManagerSaml.mjs | 60 +++++++ .../app/src/InitSamlSettings.mjs | 16 ++ .../app/src/SamlNonCsrfRouter.mjs | 12 ++ .../app/src/SamlRouter.mjs | 14 ++ .../app/src/SamlStrategy.mjs | 62 +++++++ .../web/modules/saml-authentication/index.mjs | 26 +++ 25 files changed, 944 insertions(+), 7 deletions(-) create mode 100644 patches/@node-saml+node-saml+4.0.5.patch create mode 100644 patches/ldapauth-fork+4.3.3.patch create mode 100644 services/web/modules/ldap-authentication/app/src/AuthenticationControllerLdap.mjs create mode 100644 services/web/modules/ldap-authentication/app/src/AuthenticationManagerLdap.mjs create mode 100644 services/web/modules/ldap-authentication/app/src/InitLdapSettings.mjs create mode 100644 services/web/modules/ldap-authentication/app/src/LdapContacts.mjs create mode 100644 services/web/modules/ldap-authentication/app/src/LdapStrategy.mjs create mode 100644 services/web/modules/ldap-authentication/index.mjs create mode 100644 services/web/modules/saml-authentication/app/src/AuthenticationControllerSaml.mjs create mode 100644 services/web/modules/saml-authentication/app/src/AuthenticationManagerSaml.mjs create mode 100644 services/web/modules/saml-authentication/app/src/InitSamlSettings.mjs create mode 100644 services/web/modules/saml-authentication/app/src/SamlNonCsrfRouter.mjs create mode 100644 services/web/modules/saml-authentication/app/src/SamlRouter.mjs create mode 100644 services/web/modules/saml-authentication/app/src/SamlStrategy.mjs create mode 100644 services/web/modules/saml-authentication/index.mjs diff --git a/patches/@node-saml+node-saml+4.0.5.patch b/patches/@node-saml+node-saml+4.0.5.patch new file mode 100644 index 00000000000..81fd700b319 --- /dev/null +++ b/patches/@node-saml+node-saml+4.0.5.patch @@ -0,0 +1,23 @@ +diff --git a/node_modules/@node-saml/node-saml/lib/saml.js b/node_modules/@node-saml/node-saml/lib/saml.js +index fba15b9..a5778cb 100644 +--- a/node_modules/@node-saml/node-saml/lib/saml.js ++++ b/node_modules/@node-saml/node-saml/lib/saml.js +@@ -336,7 +336,8 @@ class SAML { + const requestOrResponse = request || response; + (0, utility_1.assertRequired)(requestOrResponse, "either request or response is required"); + let buffer; +- if (this.options.skipRequestCompression) { ++ // logout requestOrResponse must be compressed anyway ++ if (this.options.skipRequestCompression && operation !== "logout") { + buffer = Buffer.from(requestOrResponse, "utf8"); + } + else { +@@ -495,7 +496,7 @@ class SAML { + try { + xml = Buffer.from(container.SAMLResponse, "base64").toString("utf8"); + doc = await (0, xml_1.parseDomFromString)(xml); +- const inResponseToNodes = xml_1.xpath.selectAttributes(doc, "/*[local-name()='Response']/@InResponseTo"); ++ const inResponseToNodes = xml_1.xpath.selectAttributes(doc, "/*[local-name()='Response' or local-name()='LogoutResponse']/@InResponseTo"); + if (inResponseToNodes) { + inResponseTo = inResponseToNodes.length ? inResponseToNodes[0].nodeValue : null; + await this.validateInResponseTo(inResponseTo); diff --git a/patches/ldapauth-fork+4.3.3.patch b/patches/ldapauth-fork+4.3.3.patch new file mode 100644 index 00000000000..4d31210c9da --- /dev/null +++ b/patches/ldapauth-fork+4.3.3.patch @@ -0,0 +1,64 @@ +diff --git a/node_modules/ldapauth-fork/lib/ldapauth.js b/node_modules/ldapauth-fork/lib/ldapauth.js +index 85ecf36a8b..a7d07e0f78 100644 +--- a/node_modules/ldapauth-fork/lib/ldapauth.js ++++ b/node_modules/ldapauth-fork/lib/ldapauth.js +@@ -69,6 +69,7 @@ function LdapAuth(opts) { + this.opts.bindProperty || (this.opts.bindProperty = 'dn'); + this.opts.groupSearchScope || (this.opts.groupSearchScope = 'sub'); + this.opts.groupDnProperty || (this.opts.groupDnProperty = 'dn'); ++ this.opts.tlsStarted = false; + + EventEmitter.call(this); + +@@ -108,21 +109,7 @@ function LdapAuth(opts) { + this._userClient.on('error', this._handleError.bind(this)); + + var self = this; +- if (this.opts.starttls) { +- // When starttls is enabled, this callback supplants the 'connect' callback +- this._adminClient.starttls(this.opts.tlsOptions, this._adminClient.controls, function(err) { +- if (err) { +- self._handleError(err); +- } else { +- self._onConnectAdmin(); +- } +- }); +- this._userClient.starttls(this.opts.tlsOptions, this._userClient.controls, function(err) { +- if (err) { +- self._handleError(err); +- } +- }); +- } else if (opts.reconnect) { ++ if (opts.reconnect && !this.opts.starttls) { + this.once('_installReconnectListener', function() { + self.log && self.log.trace('install reconnect listener'); + self._adminClient.on('connect', function() { +@@ -384,6 +371,28 @@ LdapAuth.prototype._findGroups = function(user, callback) { + */ + LdapAuth.prototype.authenticate = function(username, password, callback) { + var self = this; ++ if (this.opts.starttls && !this.opts.tlsStarted) { ++ // When starttls is enabled, this callback supplants the 'connect' callback ++ this._adminClient.starttls(this.opts.tlsOptions, this._adminClient.controls, function (err) { ++ if (err) { ++ self._handleError(err); ++ } else { ++ self._onConnectAdmin(function(){self._handleAuthenticate(username, password, callback);}); ++ } ++ }); ++ this._userClient.starttls(this.opts.tlsOptions, this._userClient.controls, function (err) { ++ if (err) { ++ self._handleError(err); ++ } ++ }); ++ } else { ++ self._handleAuthenticate(username, password, callback); ++ } ++}; ++ ++LdapAuth.prototype._handleAuthenticate = function (username, password, callback) { ++ this.opts.tlsStarted = true; ++ var self = this; + + if (typeof password === 'undefined' || password === null || password === '') { + return callback(new Error('no password given')); diff --git a/services/web/app/src/Features/Authentication/AuthenticationController.mjs b/services/web/app/src/Features/Authentication/AuthenticationController.mjs index 16acb68605d..71d3c5fb270 100644 --- a/services/web/app/src/Features/Authentication/AuthenticationController.mjs +++ b/services/web/app/src/Features/Authentication/AuthenticationController.mjs @@ -101,9 +101,9 @@ const AuthenticationController = { // so we can send back our custom `{message: {text: "", type: ""}}` responses on failure, // and send a `{redir: ""}` response on success passport.authenticate( - 'local', + Settings.ldap?.enable ? ['custom-fail-ldapauth','local'] : ['local'], { keepSessionInfo: true }, - async function (err, user, info) { + async function (err, user, infoArray) { if (err) { return next(err) } @@ -125,6 +125,7 @@ const AuthenticationController = { return next(err) } } else { + let info = infoArray[0] if (info.redir != null) { return res.json({ redir: info.redir }) } else { diff --git a/services/web/app/src/Features/PasswordReset/PasswordResetController.mjs b/services/web/app/src/Features/PasswordReset/PasswordResetController.mjs index 8079e541877..5ee322c873e 100644 --- a/services/web/app/src/Features/PasswordReset/PasswordResetController.mjs +++ b/services/web/app/src/Features/PasswordReset/PasswordResetController.mjs @@ -158,6 +158,10 @@ async function requestReset(req, res, next) { return res.status(404).json({ message: req.i18n.translate('secondary_email_password_reset'), }) + } else if (status === 'external') { + return res.status(403).json({ + message: req.i18n.translate('password_managed_externally'), + }) } else { return res.status(404).json({ message: req.i18n.translate('cant_find_email'), diff --git a/services/web/app/src/Features/PasswordReset/PasswordResetHandler.mjs b/services/web/app/src/Features/PasswordReset/PasswordResetHandler.mjs index 33d785e80fe..b4594dba15f 100644 --- a/services/web/app/src/Features/PasswordReset/PasswordResetHandler.mjs +++ b/services/web/app/src/Features/PasswordReset/PasswordResetHandler.mjs @@ -18,6 +18,10 @@ async function generateAndEmailResetToken(email) { return null } + if (!user.hashedPassword) { + return 'external' + } + if (user.email !== email) { return 'secondary' } diff --git a/services/web/app/src/Features/User/UserController.mjs b/services/web/app/src/Features/User/UserController.mjs index 37cc016f9c7..485cf1fc6bc 100644 --- a/services/web/app/src/Features/User/UserController.mjs +++ b/services/web/app/src/Features/User/UserController.mjs @@ -422,7 +422,7 @@ async function updateUserSettings(req, res, next) { if ( newEmail == null || newEmail === user.email || - req.externalAuthenticationSystemUsed() + (req.externalAuthenticationSystemUsed() && !user.hashedPassword) ) { // end here, don't update email SessionManager.setInSessionUser(req.session, { @@ -509,6 +509,7 @@ async function doLogout(req) { } async function logout(req, res, next) { + if (req?.session.saml_extce) return res.redirect(308, '/saml/logout') const requestedRedirect = req.body.redirect ? UrlHelper.getSafeRedirectPath(req.body.redirect) : undefined diff --git a/services/web/app/views/user/login.pug b/services/web/app/views/user/login.pug index 5e8a23b0172..e17769d1ea1 100644 --- a/services/web/app/views/user/login.pug +++ b/services/web/app/views/user/login.pug @@ -26,8 +26,9 @@ block content label(for='email') #{translate("email")} input#email.form-control( name='email' - type='email' + type=(settings.ldap && settings.ldap.enable) ? 'text' : 'email' required + placeholder=(settings.ldap && settings.ldap.enable) ? settings.ldap.placeholder : 'email@example.com' autofocus='true' autocomplete='username' ) @@ -47,3 +48,12 @@ block content if login_support_text hr p.text-center !{login_support_text} + if settings.saml && settings.saml.enable + form(data-ol-async-form, name="samlLoginForm") + .actions(style='margin-top: 30px;') + a.btn.btn-secondary.btn-block( + href='/saml/login', + data-ol-disabled-inflight + ) + span(data-ol-inflight="idle") #{settings.saml.identityServiceName} + span(hidden data-ol-inflight="pending") #{translate("logging_in")}… diff --git a/services/web/app/views/user/settings.pug b/services/web/app/views/user/settings.pug index d690f0b4e18..f864185821b 100644 --- a/services/web/app/views/user/settings.pug +++ b/services/web/app/views/user/settings.pug @@ -11,7 +11,7 @@ block append meta meta( name='ol-shouldAllowEditingDetails' data-type='boolean' - content=shouldAllowEditingDetails + content=shouldAllowEditingDetails || hasPassword ) meta(name='ol-oauthProviders' data-type='json' content=oauthProviders) meta(name='ol-institutionLinked' data-type='json' content=institutionLinked) @@ -34,7 +34,7 @@ block append meta meta( name='ol-isExternalAuthenticationSystemUsed' data-type='boolean' - content=externalAuthenticationSystemUsed() + content=externalAuthenticationSystemUsed() && !hasPassword ) meta(name='ol-user' data-type='json' content=user) meta(name='ol-showAiFeatures' data-type='boolean' content=showAiFeatures) diff --git a/services/web/config/settings.defaults.js b/services/web/config/settings.defaults.js index 8382fbf3a7a..fa10cf40fa7 100644 --- a/services/web/config/settings.defaults.js +++ b/services/web/config/settings.defaults.js @@ -1070,6 +1070,8 @@ module.exports = { 'launchpad', 'server-ce-scripts', 'user-activate', + 'ldap-authentication', + 'saml-authentication', 'symbol-palette', 'track-changes', ], diff --git a/services/web/locales/en.json b/services/web/locales/en.json index b0837feef00..af47164f772 100644 --- a/services/web/locales/en.json +++ b/services/web/locales/en.json @@ -175,6 +175,7 @@ "already_have_sl_account": "Already have an __appName__ account?", "also": "Also", "alternatively_create_new_institution_account": "Alternatively, you can create a new account with your institution email (__email__) by clicking __clickText__.", + "alternatively_create_local_admin_account": "Alternatively, you can create __appName__ local admin account.", "an_email_has_already_been_sent_to": "An email has already been sent to <0>__email__. Please wait and try again later.", "an_error_occured_while_restoring_project": "An error occured while restoring the project", "an_error_occurred_when_verifying_the_coupon_code": "An error occurred when verifying the coupon code", @@ -1335,6 +1336,7 @@ "loading_github_repositories": "Loading your GitHub repositories", "loading_prices": "loading prices", "loading_recent_github_commits": "Loading recent commits", + "local_account": "Local account", "log_entry_description": "Log entry with level: __level__", "log_entry_maximum_entries": "Maximum log entries limit hit", "log_entry_maximum_entries_enable_stop_on_first_error": "Try to fix the first error and recompile. Often one error causes many later error messages. You can <0>Enable “Stop on first error” to focus on fixing errors. We recommend fixing errors as soon as possible; letting them accumulate may lead to hard-to-debug and fatal errors. <1>Learn more", diff --git a/services/web/modules/launchpad/app/src/LaunchpadController.mjs b/services/web/modules/launchpad/app/src/LaunchpadController.mjs index 94cc8b35a25..5e2add12677 100644 --- a/services/web/modules/launchpad/app/src/LaunchpadController.mjs +++ b/services/web/modules/launchpad/app/src/LaunchpadController.mjs @@ -153,7 +153,8 @@ function registerExternalAuthAdmin(authMethod) { await User.updateOne( { _id: user._id }, { - $set: { isAdmin: true, emails: [{ email, reversedHostname }] }, + $set: { isAdmin: true, emails: [{ email, reversedHostname, 'confirmedAt' : Date.now() }] }, + $unset: { 'hashedPassword': "" }, // external-auth user must not have a hashedPassword } ).exec() } catch (err) { diff --git a/services/web/modules/launchpad/app/views/launchpad.pug b/services/web/modules/launchpad/app/views/launchpad.pug index 1af19bb4feb..f7b15d90627 100644 --- a/services/web/modules/launchpad/app/views/launchpad.pug +++ b/services/web/modules/launchpad/app/views/launchpad.pug @@ -129,6 +129,41 @@ block content span(data-ol-inflight='idle') #{translate("register")} span(hidden data-ol-inflight='pending') #{translate("registering")}… + h3 #{translate('local_account')} + p + | #{translate('alternatively_create_local_admin_account')} + + form( + data-ol-async-form + data-ol-register-admin + action='/launchpad/register_admin' + method='POST' + ) + input(name='_csrf' type='hidden' value=csrfToken) + +formMessages + .form-group + label.form-label(for='email') #{translate("email")} + input.form-control( + name='email' + type='email' + id='email-local' + autocomplete='username' + required + autofocus='true' + ) + .form-group + label.form-label(for='passwordField') #{translate("password")} + input#passwordField.form-control( + name='password' + type='password' + autocomplete='new-password' + required + ) + .actions + button.btn-primary.btn(type='submit' data-ol-disabled-inflight) + span(data-ol-inflight='idle') #{translate("register")} + span(hidden data-ol-inflight='pending') #{translate("registering")}… + // Saml Form if authMethod === 'saml' h3 #{translate('saml')} @@ -158,6 +193,41 @@ block content span(data-ol-inflight='idle') #{translate("register")} span(hidden data-ol-inflight='pending') #{translate("registering")}… + h3 #{translate('local_account')} + p + | #{translate('alternatively_create_local_admin_account')} + + form( + data-ol-async-form + data-ol-register-admin + action='/launchpad/register_admin' + method='POST' + ) + input(name='_csrf' type='hidden' value=csrfToken) + +formMessages + .form-group + label.form-label(for='email-local') #{translate("email")} + input.form-control( + name='email' + type='email' + id='email-local' + autocomplete='username' + required + autofocus='true' + ) + .form-group + label.form-label(for='passwordField') #{translate("password")} + input#passwordField.form-control( + name='password' + type='password' + autocomplete='new-password' + required + ) + .actions + button.btn-primary.btn(type='submit' data-ol-disabled-inflight) + span(data-ol-inflight='idle') #{translate("register")} + span(hidden data-ol-inflight='pending') #{translate("registering")}… + br diff --git a/services/web/modules/ldap-authentication/app/src/AuthenticationControllerLdap.mjs b/services/web/modules/ldap-authentication/app/src/AuthenticationControllerLdap.mjs new file mode 100644 index 00000000000..64fa4f5a96f --- /dev/null +++ b/services/web/modules/ldap-authentication/app/src/AuthenticationControllerLdap.mjs @@ -0,0 +1,64 @@ +import logger from '@overleaf/logger' +import LoginRateLimiter from '../../../../app/src/Features/Security/LoginRateLimiter.js' +import { handleAuthenticateErrors } from '../../../../app/src/Features/Authentication/AuthenticationErrors.js' +import AuthenticationController from '../../../../app/src/Features/Authentication/AuthenticationController.js' +import AuthenticationManagerLdap from './AuthenticationManagerLdap.mjs' + +const AuthenticationControllerLdap = { + async doPassportLdapLogin(req, ldapUser, done) { + let user, info + try { + ;({ user, info } = await AuthenticationControllerLdap._doPassportLdapLogin( + req, + ldapUser + )) + } catch (error) { + return done(error) + } + return done(undefined, user, info) + }, + async _doPassportLdapLogin(req, ldapUser) { + const { fromKnownDevice } = AuthenticationController.getAuditInfo(req) + const auditLog = { + ipAddress: req.ip, + info: { method: 'LDAP password login', fromKnownDevice }, + } + + let user, isPasswordReused + try { + user = await AuthenticationManagerLdap.promises.findOrCreateLdapUser(ldapUser, auditLog) + } catch (error) { + return { + user: false, + info: handleAuthenticateErrors(error, req), + } + } + if (user && AuthenticationController.captchaRequiredForLogin(req, user)) { + return { + user: false, + info: { + text: req.i18n.translate('cannot_verify_user_not_robot'), + type: 'error', + errorReason: 'cannot_verify_user_not_robot', + status: 400, + }, + } + } else if (user) { + // async actions + return { user, info: undefined } + } else { //something wrong + logger.debug({ email : ldapUser.mail }, 'failed LDAP log in') + return { + user: false, + info: { + type: 'error', + status: 500, + }, + } + } + }, +} + +export const { + doPassportLdapLogin, +} = AuthenticationControllerLdap diff --git a/services/web/modules/ldap-authentication/app/src/AuthenticationManagerLdap.mjs b/services/web/modules/ldap-authentication/app/src/AuthenticationManagerLdap.mjs new file mode 100644 index 00000000000..1371f76d525 --- /dev/null +++ b/services/web/modules/ldap-authentication/app/src/AuthenticationManagerLdap.mjs @@ -0,0 +1,80 @@ +import Settings from '@overleaf/settings' +import { callbackify } from '@overleaf/promise-utils' +import UserCreator from '../../../../app/src/Features/User/UserCreator.js' +import { User } from '../../../../app/src/models/User.js' + +const AuthenticationManagerLdap = { + splitFullName(fullName) { + fullName = fullName.trim(); + let lastSpaceIndex = fullName.lastIndexOf(' '); + let firstNames = fullName.substring(0, lastSpaceIndex).trim(); + let lastName = fullName.substring(lastSpaceIndex + 1).trim(); + return [firstNames, lastName]; + }, + async findOrCreateLdapUser(profile, auditLog) { + //user is already authenticated in Ldap + const { + attEmail, + attFirstName, + attLastName, + attName, + attAdmin, + valAdmin, + updateUserDetailsOnLogin, + } = Settings.ldap + + const email = Array.isArray(profile[attEmail]) + ? profile[attEmail][0].toLowerCase() + : profile[attEmail].toLowerCase() + let nameParts = ["",""] + if ((!attFirstName || !attLastName) && attName) { + nameParts = this.splitFullName(profile[attName] || "") + } + const firstName = attFirstName ? (profile[attFirstName] || "") : nameParts[0] + let lastName = attLastName ? (profile[attLastName] || "") : nameParts[1] + if (!firstName && !lastName) lastName = email + let isAdmin = false + if( attAdmin && valAdmin ) { + isAdmin = (profile._groups?.length > 0) || + (Array.isArray(profile[attAdmin]) ? profile[attAdmin].includes(valAdmin) : + profile[attAdmin] === valAdmin) + } + let user = await User.findOne({ 'email': email }).exec() + if( !user ) { + user = await UserCreator.promises.createNewUser( + { + email: email, + first_name: firstName, + last_name: lastName, + isAdmin: isAdmin, + holdingAccount: false, + } + ) + await User.updateOne( + { _id: user._id }, + { $set : { 'emails.0.confirmedAt' : Date.now() } } + ).exec() //email of ldap user is confirmed + } + let userDetails = updateUserDetailsOnLogin ? { first_name : firstName, last_name: lastName } : {} + if( attAdmin && valAdmin ) { + user.isAdmin = isAdmin + userDetails.isAdmin = isAdmin + } + const result = await User.updateOne( + { _id: user._id, loginEpoch: user.loginEpoch }, { $inc: { loginEpoch: 1 }, $set: userDetails }, + {} + ).exec() + if (result.modifiedCount !== 1) { + throw new ParallelLoginError() + } + return user + }, +} + +export default { + findOrCreateLdapUser: callbackify(AuthenticationManagerLdap.findOrCreateLdapUser), + promises: AuthenticationManagerLdap, +} +export const { + splitFullName, +} = AuthenticationManagerLdap diff --git a/services/web/modules/ldap-authentication/app/src/InitLdapSettings.mjs b/services/web/modules/ldap-authentication/app/src/InitLdapSettings.mjs new file mode 100644 index 00000000000..e7f312fc113 --- /dev/null +++ b/services/web/modules/ldap-authentication/app/src/InitLdapSettings.mjs @@ -0,0 +1,17 @@ +import Settings from '@overleaf/settings' + +function initLdapSettings() { + Settings.ldap = { + enable: true, + placeholder: process.env.OVERLEAF_LDAP_PLACEHOLDER || 'Username', + attEmail: process.env.OVERLEAF_LDAP_EMAIL_ATT || 'mail', + attFirstName: process.env.OVERLEAF_LDAP_FIRST_NAME_ATT, + attLastName: process.env.OVERLEAF_LDAP_LAST_NAME_ATT, + attName: process.env.OVERLEAF_LDAP_NAME_ATT, + attAdmin: process.env.OVERLEAF_LDAP_IS_ADMIN_ATT, + valAdmin: process.env.OVERLEAF_LDAP_IS_ADMIN_ATT_VALUE, + updateUserDetailsOnLogin: String(process.env.OVERLEAF_LDAP_UPDATE_USER_DETAILS_ON_LOGIN ).toLowerCase() === 'true', + } +} + +export default initLdapSettings diff --git a/services/web/modules/ldap-authentication/app/src/LdapContacts.mjs b/services/web/modules/ldap-authentication/app/src/LdapContacts.mjs new file mode 100644 index 00000000000..c4093b8684e --- /dev/null +++ b/services/web/modules/ldap-authentication/app/src/LdapContacts.mjs @@ -0,0 +1,136 @@ +import Settings from '@overleaf/settings' +import logger from '@overleaf/logger' +import passport from 'passport' +import ldapjs from 'ldapauth-fork/node_modules/ldapjs/lib/index.js' +import UserGetter from '../../../../app/src/Features/User/UserGetter.js' +import { splitFullName } from './AuthenticationManagerLdap.mjs' + +async function fetchLdapContacts(userId, contacts) { + if (!Settings.ldap?.enable || !process.env.OVERLEAF_LDAP_CONTACTS_FILTER) { + return [] + } + + const ldapOpts = passport._strategy('custom-fail-ldapauth').options.server + const { attEmail, attFirstName = "", attLastName = "", attName = "" } = Settings.ldap + const { + url, + timeout, + connectTimeout, + tlsOptions, + starttls, + bindDN, + bindCredentials, + } = ldapOpts + const searchBase = process.env.OVERLEAF_LDAP_CONTACTS_SEARCH_BASE || ldapOpts.searchBase + const searchScope = process.env.OVERLEAF_LDAP_CONTACTS_SEARCH_SCOPE || 'sub' + const ldapConfig = { url, timeout, connectTimeout, tlsOptions } + + let ldapUsers + const client = ldapjs.createClient(ldapConfig) + try { + if (starttls) { + await _upgradeToTLS(client, tlsOptions) + } + await _bindLdap(client, bindDN, bindCredentials) + + const filter = await _formContactsSearchFilter(client, ldapOpts, userId, process.env.OVERLEAF_LDAP_CONTACTS_FILTER) + const searchOptions = { scope: searchScope, attributes: [attEmail, attFirstName, attLastName, attName], filter } + + ldapUsers = await _searchLdap(client, searchBase, searchOptions) + } catch (err) { + logger.warn({ err }, 'error in fetchLdapContacts') + return [] + } finally { + client.unbind() + } + + const newLdapContacts = ldapUsers.reduce((acc, ldapUser) => { + const email = Array.isArray(ldapUser[attEmail]) + ? ldapUser[attEmail][0]?.toLowerCase() + : ldapUser[attEmail]?.toLowerCase() + if (!email) return acc + if (!contacts.some(contact => contact.email === email)) { + let nameParts = ["",""] + if ((!attFirstName || !attLastName) && attName) { + nameParts = splitFullName(ldapUser[attName] || "") + } + const firstName = attFirstName ? (ldapUser[attFirstName] || "") : nameParts[0] + const lastName = attLastName ? (ldapUser[attLastName] || "") : nameParts[1] + acc.push({ + first_name: firstName, + last_name: lastName, + email: email, + type: 'user', + }) + } + return acc + }, []) + + return newLdapContacts.sort((a, b) => + a.last_name.localeCompare(b.last_name) || + a.first_name.localeCompare(a.first_name) || + a.email.localeCompare(b.email) + ) +} + +function _upgradeToTLS(client, tlsOptions) { + return new Promise((resolve, reject) => { + client.on('error', error => reject(new Error(`LDAP client error: ${error}`))) + client.on('connect', () => { + client.starttls(tlsOptions, null, error => { + if (error) { + reject(new Error(`StartTLS error: ${error}`)) + } else { + resolve() + } + }) + }) + }) +} + +function _bindLdap(client, bindDN, bindCredentials) { + return new Promise((resolve, reject) => { + client.bind(bindDN, bindCredentials, error => { + if (error) { + reject(error) + } else { + resolve() + } + }) + }) +} + +function _searchLdap(client, baseDN, options) { + return new Promise((resolve, reject) => { + const searchEntries = [] + client.search(baseDN, options, (error, res) => { + if (error) { + reject(error) + } else { + res.on('searchEntry', entry => searchEntries.push(entry.object)) + res.on('error', reject) + res.on('end', () => resolve(searchEntries)) + } + }) + }) +} + +async function _formContactsSearchFilter(client, ldapOpts, userId, contactsFilter) { + const searchProperty = process.env.OVERLEAF_LDAP_CONTACTS_PROPERTY + if (!searchProperty) { + return contactsFilter + } + const email = await UserGetter.promises.getUserEmail(userId) + const searchOptions = { + scope: ldapOpts.searchScope, + attributes: [searchProperty], + filter: `(${Settings.ldap.attEmail}=${email})`, + } + const searchBase = ldapOpts.searchBase + const ldapUser = (await _searchLdap(client, searchBase, searchOptions))[0] + const searchPropertyValue = ldapUser ? ldapUser[searchProperty] + : process.env.OVERLEAF_LDAP_CONTACTS_NON_LDAP_VALUE || 'IMATCHNOTHING' + return contactsFilter.replace(/{{userProperty}}/g, searchPropertyValue) +} + +export default fetchLdapContacts diff --git a/services/web/modules/ldap-authentication/app/src/LdapStrategy.mjs b/services/web/modules/ldap-authentication/app/src/LdapStrategy.mjs new file mode 100644 index 00000000000..b07dc3f3bdd --- /dev/null +++ b/services/web/modules/ldap-authentication/app/src/LdapStrategy.mjs @@ -0,0 +1,78 @@ +import fs from 'fs' +import passport from 'passport' +import Settings from '@overleaf/settings' +import { doPassportLdapLogin } from './AuthenticationControllerLdap.mjs' +import { Strategy as LdapStrategy } from 'passport-ldapauth' + +function _readFilesContentFromEnv(envVar) { +// envVar is either a file name: 'file.pem', or string with array: '["file.pem", "file2.pem"]' + if (!envVar) return undefined + try { + const parsedFileNames = JSON.parse(envVar) + return parsedFileNames.map(filename => fs.readFileSync(filename, 'utf8')) + } catch (error) { + if (error instanceof SyntaxError) { // failed to parse, envVar must be a file name + return fs.readFileSync(envVar, 'utf8') + } else { + throw error + } + } +} + +// custom responses on authentication failure +class CustomFailLdapStrategy extends LdapStrategy { + constructor(options, validate) { + super(options, validate); + this.name = 'custom-fail-ldapauth' + } + authenticate(req, options) { + const defaultFail = this.fail.bind(this) + this.fail = function(info, status) { + info.type = 'error' + info.key = 'invalid-password-retry-or-reset' + info.status = 401 + return defaultFail(info, status) + }.bind(this) + super.authenticate(req, options) + } +} + +const ldapServerOpts = { + url: process.env.OVERLEAF_LDAP_URL, + bindDN: process.env.OVERLEAF_LDAP_BIND_DN || "", + bindCredentials: process.env.OVERLEAF_LDAP_BIND_CREDENTIALS || "", + bindProperty: process.env.OVERLEAF_LDAP_BIND_PROPERTY, + searchBase: process.env.OVERLEAF_LDAP_SEARCH_BASE, + searchFilter: process.env.OVERLEAF_LDAP_SEARCH_FILTER, + searchScope: process.env.OVERLEAF_LDAP_SEARCH_SCOPE || 'sub', + searchAttributes: JSON.parse(process.env.OVERLEAF_LDAP_SEARCH_ATTRIBUTES || '[]'), + groupSearchBase: process.env.OVERLEAF_LDAP_ADMIN_SEARCH_BASE, + groupSearchFilter: process.env.OVERLEAF_LDAP_ADMIN_SEARCH_FILTER, + groupSearchScope: process.env.OVERLEAF_LDAP_ADMIN_SEARCH_SCOPE || 'sub', + groupSearchAttributes: ["dn"], + groupDnProperty: process.env.OVERLEAF_LDAP_ADMIN_DN_PROPERTY, + cache: String(process.env.OVERLEAF_LDAP_CACHE).toLowerCase() === 'true', + timeout: process.env.OVERLEAF_LDAP_TIMEOUT ? Number(process.env.OVERLEAF_LDAP_TIMEOUT) : undefined, + connectTimeout: process.env.OVERLEAF_LDAP_CONNECT_TIMEOUT ? Number(process.env.OVERLEAF_LDAP_CONNECT_TIMEOUT) : undefined, + starttls: String(process.env.OVERLEAF_LDAP_STARTTLS).toLowerCase() === 'true', + tlsOptions: { + ca: _readFilesContentFromEnv(process.env.OVERLEAF_LDAP_TLS_OPTS_CA_PATH), + rejectUnauthorized: String(process.env.OVERLEAF_LDAP_TLS_OPTS_REJECT_UNAUTH).toLowerCase() === 'true', + } +} + +function addLdapStrategy(passport) { + passport.use( + new CustomFailLdapStrategy( + { + server: ldapServerOpts, + passReqToCallback: true, + usernameField: 'email', + passwordField: 'password', + }, + doPassportLdapLogin + ) + ) +} + +export default addLdapStrategy diff --git a/services/web/modules/ldap-authentication/index.mjs b/services/web/modules/ldap-authentication/index.mjs new file mode 100644 index 00000000000..f56d7ffee08 --- /dev/null +++ b/services/web/modules/ldap-authentication/index.mjs @@ -0,0 +1,30 @@ +import initLdapSettings from './app/src/InitLdapSettings.mjs' +import addLdapStrategy from './app/src/LdapStrategy.mjs' +import fetchLdapContacts from './app/src/LdapContacts.mjs' + +let ldapModule = {}; +if (process.env.EXTERNAL_AUTH === 'ldap') { + initLdapSettings() + ldapModule = { + name: 'ldap-authentication', + hooks: { + passportSetup: function (passport, callback) { + try { + addLdapStrategy(passport) + callback(null) + } catch (error) { + callback(error) + } + }, + getContacts: async function (userId, contacts, callback) { + try { + const newLdapContacts = await fetchLdapContacts(userId, contacts) + callback(null, newLdapContacts) + } catch (error) { + callback(error) + } + }, + } + } +} +export default ldapModule diff --git a/services/web/modules/saml-authentication/app/src/AuthenticationControllerSaml.mjs b/services/web/modules/saml-authentication/app/src/AuthenticationControllerSaml.mjs new file mode 100644 index 00000000000..f5db3f738dd --- /dev/null +++ b/services/web/modules/saml-authentication/app/src/AuthenticationControllerSaml.mjs @@ -0,0 +1,160 @@ +import Settings from '@overleaf/settings' +import logger from '@overleaf/logger' +import passport from 'passport' +import AuthenticationController from '../../../../app/src/Features/Authentication/AuthenticationController.js' +import AuthenticationManagerSaml from './AuthenticationManagerSaml.mjs' +import UserController from '../../../../app/src/Features/User/UserController.js' +import UserSessionsManager from '../../../../app/src/Features/User/UserSessionsManager.js' +import { handleAuthenticateErrors } from '../../../../app/src/Features/Authentication/AuthenticationErrors.js' +import { xmlResponse } from '../../../../app/src/infrastructure/Response.js' + +const AuthenticationControllerSaml = { + passportSamlAuthWithIdP(req, res, next) { + if ( passport._strategy('saml')._saml.options.authnRequestBinding === 'HTTP-POST') { + const csp = res.getHeader('Content-Security-Policy') + if (csp) { + res.setHeader( + 'Content-Security-Policy', + csp.replace(/(?:^|\s)(default-src|form-action)[^;]*;?/g, '') + ) + } + } + passport.authenticate('saml')(req, res, next) + }, + passportSamlLogin(req, res, next) { + // This function is middleware which wraps the passport.authenticate middleware, + // so we can send back our custom `{message: {text: "", type: ""}}` responses on failure, + // and send a `{redir: ""}` response on success + passport.authenticate( + 'saml', + { keepSessionInfo: true }, + async function (err, user, info) { + if (err) { + return next(err) + } + if (user) { + // `user` is either a user object or false + AuthenticationController.setAuditInfo(req, { + method: 'SAML login', + }) + try { + await AuthenticationController.promises.finishLogin(user, req, res) + } catch (err) { + return next(err) + } + } else { + if (info.redir != null) { + return res.json({ redir: info.redir }) + } else { + res.status(info.status || 200) + delete info.status + const body = { message: info } + const { errorReason } = info + if (errorReason) { + body.errorReason = errorReason + delete info.errorReason + } + return res.json(body) + } + } + } + )(req, res, next) + }, + async doPassportSamlLogin(req, profile, done) { + let user, info + try { + ;({ user, info } = await AuthenticationControllerSaml._doPassportSamlLogin( + req, + profile + )) + } catch (error) { + return done(error) + } + return done(undefined, user, info) + }, + async _doPassportSamlLogin(req, profile) { + const { fromKnownDevice } = AuthenticationController.getAuditInfo(req) + const auditLog = { + ipAddress: req.ip, + info: { method: 'SAML login', fromKnownDevice }, + } + + let user + try { + user = await AuthenticationManagerSaml.promises.findOrCreateSamlUser(profile, auditLog) + } catch (error) { + return { + user: false, + info: handleAuthenticateErrors(error, req), + } + } + if (user) { + req.session.saml_extce = {nameID : profile.nameID, sessionIndex : profile.sessionIndex} + return { user, info: undefined } + } else { //something wrong + logger.debug({ email : profile.mail }, 'failed SAML log in') + return { + user: false, + info: { + type: 'error', + text: 'Unknown error', + status: 500, + }, + } + } + }, + async passportSamlSPLogout(req, res, next) { + passport._strategy('saml').logout(req, async (err, url) => { + if (err) logger.error({ err }, 'can not generate logout url') + await UserController.promises.doLogout(req) + res.redirect(url) + }) + }, + passportSamlIdPLogout(req, res, next) { + passport.authenticate('saml')(req, res, (err) => { + if (err) return next(err) + res.redirect('/login'); + }) + }, + async doPassportSamlLogout(req, profile, done) { + let user, info + try { + ;({ user, info } = await AuthenticationControllerSaml._doPassportSamlLogout( + req, + profile + )) + } catch (error) { + return done(error) + } + return done(undefined, user, info) + }, + async _doPassportSamlLogout(req, profile) { + if (req?.session?.saml_extce?.nameID === profile.nameID && + req?.session?.saml_extce?.sessionIndex === profile.sessionIndex) { + profile = req.user + } + await UserSessionsManager.promises.untrackSession(req.user, req.sessionID).catch(err => { + logger.warn({ err, userId: req.user._id }, 'failed to untrack session') + }) + return { user: profile, info: undefined } + }, + passportSamlMetadata(req, res) { + const samlStratery = passport._strategy('saml') + res.setHeader('Content-Disposition', `attachment; filename="${samlStratery._saml.options.issuer}-meta.xml"`) + xmlResponse(res, + samlStratery.generateServiceProviderMetadata( + samlStratery._saml.options.decryptionCert, + samlStratery._saml.options.signingCert + ) + ) + }, +} +export const { + passportSamlAuthWithIdP, + passportSamlLogin, + passportSamlSPLogout, + passportSamlIdPLogout, + doPassportSamlLogin, + doPassportSamlLogout, + passportSamlMetadata, +} = AuthenticationControllerSaml diff --git a/services/web/modules/saml-authentication/app/src/AuthenticationManagerSaml.mjs b/services/web/modules/saml-authentication/app/src/AuthenticationManagerSaml.mjs new file mode 100644 index 00000000000..47d97f3019c --- /dev/null +++ b/services/web/modules/saml-authentication/app/src/AuthenticationManagerSaml.mjs @@ -0,0 +1,60 @@ +import Settings from '@overleaf/settings' +import UserCreator from '../../../../app/src/Features/User/UserCreator.js' +import { User } from '../../../../app/src/models/User.js' + +const AuthenticationManagerSaml = { + async findOrCreateSamlUser(profile, auditLog) { + const { + attEmail, + attFirstName, + attLastName, + attAdmin, + valAdmin, + updateUserDetailsOnLogin, + } = Settings.saml + const email = Array.isArray(profile[attEmail]) + ? profile[attEmail][0].toLowerCase() + : profile[attEmail].toLowerCase() + const firstName = attFirstName ? profile[attFirstName] : "" + const lastName = attLastName ? profile[attLastName] : email + let isAdmin = false + if( attAdmin && valAdmin ) { + isAdmin = (Array.isArray(profile[attAdmin]) ? profile[attAdmin].includes(valAdmin) : + profile[attAdmin] === valAdmin) + } + let user = await User.findOne({ 'email': email }).exec() + if( !user ) { + user = await UserCreator.promises.createNewUser( + { + email: email, + first_name: firstName, + last_name: lastName, + isAdmin: isAdmin, + holdingAccount: false, + } + ) + await User.updateOne( + { _id: user._id }, + { $set : { 'emails.0.confirmedAt' : Date.now() } } + ).exec() //email of saml user is confirmed + } + let userDetails = updateUserDetailsOnLogin ? { first_name : firstName, last_name: lastName } : {} + if( attAdmin && valAdmin ) { + user.isAdmin = isAdmin + userDetails.isAdmin = isAdmin + } + const result = await User.updateOne( + { _id: user._id, loginEpoch: user.loginEpoch }, { $inc: { loginEpoch: 1 }, $set: userDetails }, + {} + ).exec() + + if (result.modifiedCount !== 1) { + throw new ParallelLoginError() + } + return user + }, +} + +export default { + promises: AuthenticationManagerSaml, +} diff --git a/services/web/modules/saml-authentication/app/src/InitSamlSettings.mjs b/services/web/modules/saml-authentication/app/src/InitSamlSettings.mjs new file mode 100644 index 00000000000..441f9033af9 --- /dev/null +++ b/services/web/modules/saml-authentication/app/src/InitSamlSettings.mjs @@ -0,0 +1,16 @@ +import Settings from '@overleaf/settings' + +function initSamlSettings() { + Settings.saml = { + enable: true, + identityServiceName: process.env.OVERLEAF_SAML_IDENTITY_SERVICE_NAME || 'Login with SAML IdP', + attEmail: process.env.OVERLEAF_SAML_EMAIL_FIELD || 'nameID', + attFirstName: process.env.OVERLEAF_SAML_FIRST_NAME_FIELD || 'givenName', + attLastName: process.env.OVERLEAF_SAML_LAST_NAME_FIELD || 'lastName', + attAdmin: process.env.OVERLEAF_SAML_IS_ADMIN_FIELD, + valAdmin: process.env.OVERLEAF_SAML_IS_ADMIN_FIELD_VALUE, + updateUserDetailsOnLogin: String(process.env.OVERLEAF_SAML_UPDATE_USER_DETAILS_ON_LOGIN).toLowerCase() === 'true', + } +} + +export default initSamlSettings diff --git a/services/web/modules/saml-authentication/app/src/SamlNonCsrfRouter.mjs b/services/web/modules/saml-authentication/app/src/SamlNonCsrfRouter.mjs new file mode 100644 index 00000000000..65b42c92aef --- /dev/null +++ b/services/web/modules/saml-authentication/app/src/SamlNonCsrfRouter.mjs @@ -0,0 +1,12 @@ +import logger from '@overleaf/logger' +import { passportSamlLogin, passportSamlIdPLogout } from './AuthenticationControllerSaml.mjs' + +export default { + apply(webRouter) { + logger.debug({}, 'Init SAML NonCsrfRouter') + webRouter.get('/saml/login/callback', passportSamlLogin) + webRouter.post('/saml/login/callback', passportSamlLogin) + webRouter.get('/saml/logout/callback', passportSamlIdPLogout) + webRouter.post('/saml/logout/callback', passportSamlIdPLogout) + }, +} diff --git a/services/web/modules/saml-authentication/app/src/SamlRouter.mjs b/services/web/modules/saml-authentication/app/src/SamlRouter.mjs new file mode 100644 index 00000000000..9ee36779011 --- /dev/null +++ b/services/web/modules/saml-authentication/app/src/SamlRouter.mjs @@ -0,0 +1,14 @@ +import logger from '@overleaf/logger' +import AuthenticationController from '../../../../app/src/Features/Authentication/AuthenticationController.js' +import { passportSamlAuthWithIdP, passportSamlSPLogout, passportSamlMetadata} from './AuthenticationControllerSaml.mjs' + +export default { + apply(webRouter) { + logger.debug({}, 'Init SAML router') + webRouter.get('/saml/login', passportSamlAuthWithIdP) + AuthenticationController.addEndpointToLoginWhitelist('/saml/login') + webRouter.post('/saml/logout', AuthenticationController.requireLogin(), passportSamlSPLogout) + webRouter.get('/saml/meta', passportSamlMetadata) + AuthenticationController.addEndpointToLoginWhitelist('/saml/meta') + }, +} diff --git a/services/web/modules/saml-authentication/app/src/SamlStrategy.mjs b/services/web/modules/saml-authentication/app/src/SamlStrategy.mjs new file mode 100644 index 00000000000..3a16459f981 --- /dev/null +++ b/services/web/modules/saml-authentication/app/src/SamlStrategy.mjs @@ -0,0 +1,62 @@ +import fs from 'fs' +import passport from 'passport' +import Settings from '@overleaf/settings' +import { doPassportSamlLogin, doPassportSamlLogout } from './AuthenticationControllerSaml.mjs' +import { Strategy as SamlStrategy } from '@node-saml/passport-saml' + +function _readFilesContentFromEnv(envVar) { +// envVar is either a file name: 'file.pem', or string with array: '["file.pem", "file2.pem"]' + if (!envVar) return undefined + try { + const parsedFileNames = JSON.parse(envVar) + return parsedFileNames.map(filename => fs.readFileSync(filename, 'utf8')) + } catch (error) { + if (error instanceof SyntaxError) { // failed to parse, envVar must be a file name + return fs.readFileSync(envVar, 'utf8') + } else { + throw error + } + } +} + +const samlOptions = { + entryPoint: process.env.OVERLEAF_SAML_ENTRYPOINT, + callbackUrl: process.env.OVERLEAF_SAML_CALLBACK_URL, + issuer: process.env.OVERLEAF_SAML_ISSUER, + audience: process.env.OVERLEAF_SAML_AUDIENCE, + cert: _readFilesContentFromEnv(process.env.OVERLEAF_SAML_IDP_CERT), + signingCert: _readFilesContentFromEnv(process.env.OVERLEAF_SAML_PUBLIC_CERT), + privateKey: _readFilesContentFromEnv(process.env.OVERLEAF_SAML_PRIVATE_KEY), + decryptionCert: _readFilesContentFromEnv(process.env.OVERLEAF_SAML_DECRYPTION_CERT), + decryptionPvk: _readFilesContentFromEnv(process.env.OVERLEAF_SAML_DECRYPTION_PVK), + signatureAlgorithm: process.env.OVERLEAF_SAML_SIGNATURE_ALGORITHM, + additionalParams: JSON.parse(process.env.OVERLEAF_SAML_ADDITIONAL_PARAMS || '{}'), + additionalAuthorizeParams: JSON.parse(process.env.OVERLEAF_SAML_ADDITIONAL_AUTHORIZE_PARAMS || '{}'), + identifierFormat: process.env.OVERLEAF_SAML_IDENTIFIER_FORMAT, + acceptedClockSkewMs: process.env.OVERLEAF_SAML_ACCEPTED_CLOCK_SKEW_MS ? Number(process.env.OVERLEAF_SAML_ACCEPTED_CLOCK_SKEW_MS) : undefined, + attributeConsumingServiceIndex: process.env.OVERLEAF_SAML_ATTRIBUTE_CONSUMING_SERVICE_INDEX, + authnContext: process.env.OVERLEAF_SAML_AUTHN_CONTEXT ? JSON.parse(process.env.OVERLEAF_SAML_AUTHN_CONTEXT) : undefined, + forceAuthn: String(process.env.OVERLEAF_SAML_FORCE_AUTHN).toLowerCase() === 'true', + disableRequestedAuthnContext: String(process.env.OVERLEAF_SAML_DISABLE_REQUESTED_AUTHN_CONTEXT).toLowerCase() === 'true', + skipRequestCompression: process.env.OVERLEAF_SAML_AUTHN_REQUEST_BINDING === 'HTTP-POST', // compression should be skipped iff authnRequestBinding is POST + authnRequestBinding: process.env.OVERLEAF_SAML_AUTHN_REQUEST_BINDING, + validateInResponseTo: process.env.OVERLEAF_SAML_VALIDATE_IN_RESPONSE_TO, + requestIdExpirationPeriodMs: process.env.OVERLEAF_SAML_REQUEST_ID_EXPIRATION_PERIOD_MS ? Number(process.env.OVERLEAF_SAML_REQUEST_ID_EXPIRATION_PERIOD_MS) : undefined, +// cacheProvider: process.env.OVERLEAF_SAML_CACHE_PROVIDER, + logoutUrl: process.env.OVERLEAF_SAML_LOGOUT_URL, + logoutCallbackUrl: process.env.OVERLEAF_SAML_LOGOUT_CALLBACK_URL, + additionalLogoutParams: JSON.parse(process.env.OVERLEAF_SAML_ADDITIONAL_LOGOUT_PARAMS || '{}'), + passReqToCallback: true, +} + +function addSamlStrategy(passport) { + passport.use( + new SamlStrategy( + samlOptions, + doPassportSamlLogin, + doPassportSamlLogout + ) + ) +} + +export default addSamlStrategy diff --git a/services/web/modules/saml-authentication/index.mjs b/services/web/modules/saml-authentication/index.mjs new file mode 100644 index 00000000000..35ea70283fe --- /dev/null +++ b/services/web/modules/saml-authentication/index.mjs @@ -0,0 +1,26 @@ +import initSamlSettings from './app/src/InitSamlSettings.mjs' +import addSamlStrategy from './app/src/SamlStrategy.mjs' +import SamlRouter from './app/src/SamlRouter.mjs' +import SamlNonCsrfRouter from './app/src/SamlNonCsrfRouter.mjs' + +let samlModule = {}; + +if (process.env.EXTERNAL_AUTH === 'saml') { + initSamlSettings() + samlModule = { + name: 'saml-authentication', + hooks: { + passportSetup: function (passport, callback) { + try { + addSamlStrategy(passport) + callback(null) + } catch (error) { + callback(error) + } + }, + }, + router: SamlRouter, + nonCsrfRouter: SamlNonCsrfRouter, + } +} +export default samlModule From 53f8fd9aa366f66f77116c73317901ee37c2a5b7 Mon Sep 17 00:00:00 2001 From: yu-i-i Date: Mon, 20 Jan 2025 05:38:02 +0100 Subject: [PATCH 008/106] Allow adding extra flags to LaTeX compiler through environment variable --- services/web/app/src/Features/Compile/ClsiManager.mjs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/services/web/app/src/Features/Compile/ClsiManager.mjs b/services/web/app/src/Features/Compile/ClsiManager.mjs index 5da6c55c39f..d69e8b4cf4a 100644 --- a/services/web/app/src/Features/Compile/ClsiManager.mjs +++ b/services/web/app/src/Features/Compile/ClsiManager.mjs @@ -1018,7 +1018,7 @@ async function _getContentFromMongo(projectId) { function _finaliseRequest(projectId, options, project, docs, files) { const resources = [] - let flags + let flags = [] let rootResourcePath = options.rootResourcePath let rootResourcePathOverride = null let hasMainFile = false @@ -1087,6 +1087,10 @@ function _finaliseRequest(projectId, options, project, docs, files) { flags = ['-file-line-error'] } + if (process.env.TEX_COMPILER_EXTRA_FLAGS) { + flags.push(...process.env.TEX_COMPILER_EXTRA_FLAGS.split(/\s+/).filter(Boolean)) + } + return { compile: { options: { From 8a3b88f945bca3c4fdac807193c800edcb52217a Mon Sep 17 00:00:00 2001 From: yu-i-i Date: Tue, 17 Dec 2024 18:36:18 +0100 Subject: [PATCH 009/106] Refactor authentication code; add OIDC support --- .../AuthenticationController.mjs | 6 +- .../PasswordReset/PasswordResetController.mjs | 4 - .../PasswordReset/PasswordResetHandler.mjs | 5 +- .../app/src/Features/User/UserController.mjs | 3 +- .../src/Features/User/UserPagesController.mjs | 6 +- .../app/src/infrastructure/ExpressLocals.mjs | 4 +- services/web/app/src/router.mjs | 4 +- services/web/app/views/user/login.pug | 9 + services/web/app/views/user/passwordReset.pug | 2 +- services/web/app/views/user/settings.pug | 4 +- services/web/config/settings.defaults.js | 19 +- .../web/frontend/extracted-translations.json | 1 + .../settings/components/linking-section.tsx | 3 +- .../components/linking/sso-widget.tsx | 4 +- .../settings/components/password-section.tsx | 6 +- .../frontend/js/shared/svgs/openid-logo.jsx | 27 +++ services/web/locales/en.json | 2 + .../app/src/LDAPAuthenticationController.mjs | 112 ++++++++++++ .../app/src/LDAPAuthenticationManager.mjs} | 36 ++-- .../ldap/app/src/LDAPContacts.mjs | 120 ++++++++++++ .../ldap/app/src/LDAPModuleManager.mjs | 112 ++++++++++++ .../ldap/app/src/LDAPRouter.mjs | 19 ++ .../web/modules/authentication/ldap/index.mjs | 17 ++ .../web/modules/authentication/logout.mjs | 18 ++ .../app/src/OIDCAuthenticationController.mjs | 171 ++++++++++++++++++ .../app/src/OIDCAuthenticationManager.mjs | 94 ++++++++++ .../oidc/app/src/OIDCModuleManager.mjs | 82 +++++++++ .../oidc/app/src/OIDCRouter.mjs | 15 ++ .../web/modules/authentication/oidc/index.mjs | 16 ++ .../app/src/SAMLAuthenticationController.mjs} | 66 +++---- .../app/src/SAMLAuthenticationManager.mjs | 85 +++++++++ .../saml/app/src/SAMLModuleManager.mjs | 100 ++++++++++ .../saml/app/src/SAMLNonCsrfRouter.mjs | 11 ++ .../saml/app/src/SAMLRouter.mjs | 16 ++ .../web/modules/authentication/saml/index.mjs | 18 ++ services/web/modules/authentication/utils.mjs | 42 +++++ .../app/src/AuthenticationControllerLdap.mjs | 64 ------- .../app/src/InitLdapSettings.mjs | 17 -- .../app/src/LdapContacts.mjs | 136 -------------- .../app/src/LdapStrategy.mjs | 78 -------- .../web/modules/ldap-authentication/index.mjs | 30 --- .../app/src/AuthenticationManagerSaml.mjs | 60 ------ .../app/src/InitSamlSettings.mjs | 16 -- .../app/src/SamlNonCsrfRouter.mjs | 12 -- .../app/src/SamlRouter.mjs | 14 -- .../app/src/SamlStrategy.mjs | 62 ------- .../web/modules/saml-authentication/index.mjs | 26 --- services/web/package.json | 1 + 48 files changed, 1169 insertions(+), 606 deletions(-) create mode 100644 services/web/frontend/js/shared/svgs/openid-logo.jsx create mode 100644 services/web/modules/authentication/ldap/app/src/LDAPAuthenticationController.mjs rename services/web/modules/{ldap-authentication/app/src/AuthenticationManagerLdap.mjs => authentication/ldap/app/src/LDAPAuthenticationManager.mjs} (67%) create mode 100644 services/web/modules/authentication/ldap/app/src/LDAPContacts.mjs create mode 100644 services/web/modules/authentication/ldap/app/src/LDAPModuleManager.mjs create mode 100644 services/web/modules/authentication/ldap/app/src/LDAPRouter.mjs create mode 100644 services/web/modules/authentication/ldap/index.mjs create mode 100644 services/web/modules/authentication/logout.mjs create mode 100644 services/web/modules/authentication/oidc/app/src/OIDCAuthenticationController.mjs create mode 100644 services/web/modules/authentication/oidc/app/src/OIDCAuthenticationManager.mjs create mode 100644 services/web/modules/authentication/oidc/app/src/OIDCModuleManager.mjs create mode 100644 services/web/modules/authentication/oidc/app/src/OIDCRouter.mjs create mode 100644 services/web/modules/authentication/oidc/index.mjs rename services/web/modules/{saml-authentication/app/src/AuthenticationControllerSaml.mjs => authentication/saml/app/src/SAMLAuthenticationController.mjs} (65%) create mode 100644 services/web/modules/authentication/saml/app/src/SAMLAuthenticationManager.mjs create mode 100644 services/web/modules/authentication/saml/app/src/SAMLModuleManager.mjs create mode 100644 services/web/modules/authentication/saml/app/src/SAMLNonCsrfRouter.mjs create mode 100644 services/web/modules/authentication/saml/app/src/SAMLRouter.mjs create mode 100644 services/web/modules/authentication/saml/index.mjs create mode 100644 services/web/modules/authentication/utils.mjs delete mode 100644 services/web/modules/ldap-authentication/app/src/AuthenticationControllerLdap.mjs delete mode 100644 services/web/modules/ldap-authentication/app/src/InitLdapSettings.mjs delete mode 100644 services/web/modules/ldap-authentication/app/src/LdapContacts.mjs delete mode 100644 services/web/modules/ldap-authentication/app/src/LdapStrategy.mjs delete mode 100644 services/web/modules/ldap-authentication/index.mjs delete mode 100644 services/web/modules/saml-authentication/app/src/AuthenticationManagerSaml.mjs delete mode 100644 services/web/modules/saml-authentication/app/src/InitSamlSettings.mjs delete mode 100644 services/web/modules/saml-authentication/app/src/SamlNonCsrfRouter.mjs delete mode 100644 services/web/modules/saml-authentication/app/src/SamlRouter.mjs delete mode 100644 services/web/modules/saml-authentication/app/src/SamlStrategy.mjs delete mode 100644 services/web/modules/saml-authentication/index.mjs diff --git a/services/web/app/src/Features/Authentication/AuthenticationController.mjs b/services/web/app/src/Features/Authentication/AuthenticationController.mjs index 71d3c5fb270..00ef3f7f990 100644 --- a/services/web/app/src/Features/Authentication/AuthenticationController.mjs +++ b/services/web/app/src/Features/Authentication/AuthenticationController.mjs @@ -83,6 +83,7 @@ const AuthenticationController = { analyticsId: user.analyticsId || user._id, alphaProgram: user.alphaProgram || undefined, // only store if set betaProgram: user.betaProgram || undefined, // only store if set + externalAuth: user.externalAuth || false, } if (user.isAdmin) { lightUser.isAdmin = true @@ -101,9 +102,9 @@ const AuthenticationController = { // so we can send back our custom `{message: {text: "", type: ""}}` responses on failure, // and send a `{redir: ""}` response on success passport.authenticate( - Settings.ldap?.enable ? ['custom-fail-ldapauth','local'] : ['local'], + 'local', { keepSessionInfo: true }, - async function (err, user, infoArray) { + async function (err, user, info) { if (err) { return next(err) } @@ -125,7 +126,6 @@ const AuthenticationController = { return next(err) } } else { - let info = infoArray[0] if (info.redir != null) { return res.json({ redir: info.redir }) } else { diff --git a/services/web/app/src/Features/PasswordReset/PasswordResetController.mjs b/services/web/app/src/Features/PasswordReset/PasswordResetController.mjs index 5ee322c873e..8079e541877 100644 --- a/services/web/app/src/Features/PasswordReset/PasswordResetController.mjs +++ b/services/web/app/src/Features/PasswordReset/PasswordResetController.mjs @@ -158,10 +158,6 @@ async function requestReset(req, res, next) { return res.status(404).json({ message: req.i18n.translate('secondary_email_password_reset'), }) - } else if (status === 'external') { - return res.status(403).json({ - message: req.i18n.translate('password_managed_externally'), - }) } else { return res.status(404).json({ message: req.i18n.translate('cant_find_email'), diff --git a/services/web/app/src/Features/PasswordReset/PasswordResetHandler.mjs b/services/web/app/src/Features/PasswordReset/PasswordResetHandler.mjs index b4594dba15f..965bf42561e 100644 --- a/services/web/app/src/Features/PasswordReset/PasswordResetHandler.mjs +++ b/services/web/app/src/Features/PasswordReset/PasswordResetHandler.mjs @@ -18,10 +18,6 @@ async function generateAndEmailResetToken(email) { return null } - if (!user.hashedPassword) { - return 'external' - } - if (user.email !== email) { return 'secondary' } @@ -76,6 +72,7 @@ async function getUserForPasswordResetToken(token) { 'overleaf.id': 1, email: 1, must_reconfirm: 1, + hashedPassword: 1, }) await assertUserPermissions(user, ['change-password']) diff --git a/services/web/app/src/Features/User/UserController.mjs b/services/web/app/src/Features/User/UserController.mjs index 485cf1fc6bc..37cc016f9c7 100644 --- a/services/web/app/src/Features/User/UserController.mjs +++ b/services/web/app/src/Features/User/UserController.mjs @@ -422,7 +422,7 @@ async function updateUserSettings(req, res, next) { if ( newEmail == null || newEmail === user.email || - (req.externalAuthenticationSystemUsed() && !user.hashedPassword) + req.externalAuthenticationSystemUsed() ) { // end here, don't update email SessionManager.setInSessionUser(req.session, { @@ -509,7 +509,6 @@ async function doLogout(req) { } async function logout(req, res, next) { - if (req?.session.saml_extce) return res.redirect(308, '/saml/logout') const requestedRedirect = req.body.redirect ? UrlHelper.getSafeRedirectPath(req.body.redirect) : undefined diff --git a/services/web/app/src/Features/User/UserPagesController.mjs b/services/web/app/src/Features/User/UserPagesController.mjs index ba308bb0324..9d9c181d30a 100644 --- a/services/web/app/src/Features/User/UserPagesController.mjs +++ b/services/web/app/src/Features/User/UserPagesController.mjs @@ -53,10 +53,8 @@ async function settingsPage(req, res) { const reconfirmedViaSAML = _.get(req.session, ['saml', 'reconfirmed']) delete req.session.saml let shouldAllowEditingDetails = true - if (Settings.ldap && Settings.ldap.updateUserDetailsOnLogin) { - shouldAllowEditingDetails = false - } - if (Settings.saml && Settings.saml.updateUserDetailsOnLogin) { + const externalAuth = req.user.externalAuth + if (externalAuth && Settings[externalAuth].updateUserDetailsOnLogin) { shouldAllowEditingDetails = false } const oauthProviders = Settings.oauthProviders || {} diff --git a/services/web/app/src/infrastructure/ExpressLocals.mjs b/services/web/app/src/infrastructure/ExpressLocals.mjs index 65bb04b485c..2907f4ac08b 100644 --- a/services/web/app/src/infrastructure/ExpressLocals.mjs +++ b/services/web/app/src/infrastructure/ExpressLocals.mjs @@ -114,9 +114,9 @@ export default async function (webRouter, privateApiRouter, publicApiRouter) { webRouter.use(function (req, res, next) { req.externalAuthenticationSystemUsed = - Features.externalAuthenticationSystemUsed + () => !!req?.user?.externalAuth res.locals.externalAuthenticationSystemUsed = - Features.externalAuthenticationSystemUsed + () => !!req?.user?.externalAuth req.hasFeature = res.locals.hasFeature = Features.hasFeature next() }) diff --git a/services/web/app/src/router.mjs b/services/web/app/src/router.mjs index c17bfbb3785..41f74d79bfc 100644 --- a/services/web/app/src/router.mjs +++ b/services/web/app/src/router.mjs @@ -228,6 +228,8 @@ async function initialize(webRouter, privateApiRouter, publicApiRouter) { CaptchaMiddleware.canSkipCaptcha ) + await Modules.applyRouter(webRouter, privateApiRouter, publicApiRouter) + webRouter.get('/login', UserPagesController.loginPage) AuthenticationController.addEndpointToLoginWhitelist('/login') @@ -297,8 +299,6 @@ async function initialize(webRouter, privateApiRouter, publicApiRouter) { TokenAccessRouter.apply(webRouter) HistoryRouter.apply(webRouter, privateApiRouter) - await Modules.applyRouter(webRouter, privateApiRouter, publicApiRouter) - if (Settings.enableSubscriptions) { webRouter.get( '/user/bonus', diff --git a/services/web/app/views/user/login.pug b/services/web/app/views/user/login.pug index e17769d1ea1..bf39b8a1f72 100644 --- a/services/web/app/views/user/login.pug +++ b/services/web/app/views/user/login.pug @@ -57,3 +57,12 @@ block content ) span(data-ol-inflight="idle") #{settings.saml.identityServiceName} span(hidden data-ol-inflight="pending") #{translate("logging_in")}… + if settings.oidc && settings.oidc.enable + form(data-ol-async-form, name="oidcLoginForm") + .actions(style='margin-top: 30px;') + a.btn.btn-secondary.btn-block( + href='/oidc/login', + data-ol-disabled-inflight + ) + span(data-ol-inflight="idle") #{settings.oidc.identityServiceName} + span(hidden data-ol-inflight="pending") #{translate("logging_in")}… diff --git a/services/web/app/views/user/passwordReset.pug b/services/web/app/views/user/passwordReset.pug index ca2483c9a89..36afc189731 100644 --- a/services/web/app/views/user/passwordReset.pug +++ b/services/web/app/views/user/passwordReset.pug @@ -50,7 +50,7 @@ block content +notification({ariaLive: 'assertive', type: 'error', className: 'mb-3', content: translate(error)}) div(data-ol-custom-form-message='no-password-allowed-due-to-sso' hidden) - +notification({ariaLive: 'polite', type: 'error', className: 'mb-3', content: translate('you_cant_reset_password_due_to_sso', {}, [{name: 'a', attrs: {href: '/sso-login'}}])}) + +notification({ariaLive: 'polite', type: 'error', className: 'mb-3', content: translate('you_cant_reset_password_due_to_ldap_or_sso')}) input(name='_csrf' type='hidden' value=csrfToken) .form-group.mb-3 label.form-label(for='email') #{translate("email")} diff --git a/services/web/app/views/user/settings.pug b/services/web/app/views/user/settings.pug index f864185821b..d690f0b4e18 100644 --- a/services/web/app/views/user/settings.pug +++ b/services/web/app/views/user/settings.pug @@ -11,7 +11,7 @@ block append meta meta( name='ol-shouldAllowEditingDetails' data-type='boolean' - content=shouldAllowEditingDetails || hasPassword + content=shouldAllowEditingDetails ) meta(name='ol-oauthProviders' data-type='json' content=oauthProviders) meta(name='ol-institutionLinked' data-type='json' content=institutionLinked) @@ -34,7 +34,7 @@ block append meta meta( name='ol-isExternalAuthenticationSystemUsed' data-type='boolean' - content=externalAuthenticationSystemUsed() && !hasPassword + content=externalAuthenticationSystemUsed() ) meta(name='ol-user' data-type='json' content=user) meta(name='ol-showAiFeatures' data-type='boolean' content=showAiFeatures) diff --git a/services/web/config/settings.defaults.js b/services/web/config/settings.defaults.js index fa10cf40fa7..ce217b79fa0 100644 --- a/services/web/config/settings.defaults.js +++ b/services/web/config/settings.defaults.js @@ -1070,10 +1070,11 @@ module.exports = { 'launchpad', 'server-ce-scripts', 'user-activate', - 'ldap-authentication', - 'saml-authentication', 'symbol-palette', 'track-changes', + 'authentication/ldap', + 'authentication/saml', + 'authentication/oidc', ], viewIncludes: {}, @@ -1109,6 +1110,20 @@ module.exports = { || imageName.split(':')[1], })) : undefined, + + oauthProviders: { + ...(process.env.EXTERNAL_AUTH && process.env.EXTERNAL_AUTH.includes('oidc') && { + [process.env.OVERLEAF_OIDC_PROVIDER_ID || 'oidc']: { + name: process.env.OVERLEAF_OIDC_PROVIDER_NAME || 'OIDC Provider', + descriptionKey: process.env.OVERLEAF_OIDC_PROVIDER_DESCRIPTION, + descriptionOptions: { link: process.env.OVERLEAF_OIDC_PROVIDER_INFO_LINK }, + hideWhenNotLinked: process.env.OVERLEAF_OIDC_PROVIDER_HIDE_NOT_LINKED ? + process.env.OVERLEAF_OIDC_PROVIDER_HIDE_NOT_LINKED.toLowerCase() === 'true' : undefined, + linkPath: '/oidc/login', + }, + }), + }, + } module.exports.mergeWith = function (overrides) { diff --git a/services/web/frontend/extracted-translations.json b/services/web/frontend/extracted-translations.json index 3e887029411..9380abf2e57 100644 --- a/services/web/frontend/extracted-translations.json +++ b/services/web/frontend/extracted-translations.json @@ -2301,6 +2301,7 @@ "you_can_select_or_invite_collaborator": "", "you_can_select_or_invite_collaborator_plural": "", "you_can_still_use_your_premium_features": "", + "you_cant_add_or_change_password_due_to_ldap_or_sso": "", "you_cant_add_or_change_password_due_to_sso": "", "you_cant_join_this_group_subscription": "", "you_currently_have_x_linked_with_your_overleaf_account": "", diff --git a/services/web/frontend/js/features/settings/components/linking-section.tsx b/services/web/frontend/js/features/settings/components/linking-section.tsx index cd92ab614be..e1f53ef5ca9 100644 --- a/services/web/frontend/js/features/settings/components/linking-section.tsx +++ b/services/web/frontend/js/features/settings/components/linking-section.tsx @@ -200,7 +200,8 @@ function SSOLinkingWidgetContainer({ const { t } = useTranslation() const { unlink } = useSSOContext() - let description = '' + let description = subscription.provider.descriptionKey || + `${t('login_with_service', { service: subscription.provider.name, })}.` switch (subscription.providerId) { case 'collabratec': description = t('linked_collabratec_description') diff --git a/services/web/frontend/js/features/settings/components/linking/sso-widget.tsx b/services/web/frontend/js/features/settings/components/linking/sso-widget.tsx index c80605aefbb..0769db713e4 100644 --- a/services/web/frontend/js/features/settings/components/linking/sso-widget.tsx +++ b/services/web/frontend/js/features/settings/components/linking/sso-widget.tsx @@ -4,6 +4,7 @@ import { FetchError } from '../../../../infrastructure/fetch-json' import IEEELogo from '../../../../shared/svgs/ieee-logo' import GoogleLogo from '../../../../shared/svgs/google-logo' import OrcidLogo from '../../../../shared/svgs/orcid-logo' +import OpenIDLogo from '../../../../shared/svgs/openid-logo' import LinkingStatus from './status' import OLButton from '@/shared/components/ol/ol-button' import { @@ -18,6 +19,7 @@ const providerLogos: { readonly [p: string]: JSX.Element } = { collabratec: , google: , orcid: , + oidc: , } type SSOLinkingWidgetProps = { @@ -67,7 +69,7 @@ export function SSOLinkingWidget({ return (
-
{providerLogos[providerId]}
+
{providerLogos[providerId] || providerLogos['oidc']}

{title}

diff --git a/services/web/frontend/js/features/settings/components/password-section.tsx b/services/web/frontend/js/features/settings/components/password-section.tsx index 9128119dc38..73054dae66c 100644 --- a/services/web/frontend/js/features/settings/components/password-section.tsx +++ b/services/web/frontend/js/features/settings/components/password-section.tsx @@ -39,11 +39,7 @@ function CanOnlyLogInThroughSSO() { return (

, - ]} + i18nKey="you_cant_add_or_change_password_due_to_ldap_or_sso" />

) diff --git a/services/web/frontend/js/shared/svgs/openid-logo.jsx b/services/web/frontend/js/shared/svgs/openid-logo.jsx new file mode 100644 index 00000000000..3de933820b4 --- /dev/null +++ b/services/web/frontend/js/shared/svgs/openid-logo.jsx @@ -0,0 +1,27 @@ +function OpenIDLogo() { + return ( + + + + + + + ) +} + +export default OpenIDLogo + diff --git a/services/web/locales/en.json b/services/web/locales/en.json index af47164f772..2d68419f158 100644 --- a/services/web/locales/en.json +++ b/services/web/locales/en.json @@ -2875,8 +2875,10 @@ "you_can_select_or_invite_collaborator": "You can select or invite __count__ collaborator on your current plan. Upgrade to add more editors or reviewers.", "you_can_select_or_invite_collaborator_plural": "You can select or invite __count__ collaborators on your current plan. Upgrade to add more editors or reviewers.", "you_can_still_use_your_premium_features": "You can still use your premium features until the pause becomes active.", + "you_cant_add_or_change_password_due_to_ldap_or_sso": "You can’t add or change your password because your group or organization uses LDAP or SSO.", "you_cant_add_or_change_password_due_to_sso": "You can’t add or change your password because your group or organization uses <0>single sign-on (SSO).", "you_cant_join_this_group_subscription": "You can’t join this group subscription", + "you_cant_reset_password_due_to_ldap_or_sso": "You can’t reset your password because your group or organization uses LDAP or SSO. Contact your system administrator.", "you_cant_reset_password_due_to_sso": "You can’t reset your password because your group or organization uses SSO. <0>Log in with SSO.", "you_currently_have_x_linked_with_your_overleaf_account": "You currently have <0>__managers__ linked with your __appName__ account.", "you_dont_have_any_add_ons_on_your_account": "You don’t have any add-ons on your account.", diff --git a/services/web/modules/authentication/ldap/app/src/LDAPAuthenticationController.mjs b/services/web/modules/authentication/ldap/app/src/LDAPAuthenticationController.mjs new file mode 100644 index 00000000000..1a3ed01d3c0 --- /dev/null +++ b/services/web/modules/authentication/ldap/app/src/LDAPAuthenticationController.mjs @@ -0,0 +1,112 @@ +import logger from '@overleaf/logger' +import passport from 'passport' +import EmailHelper from '../../../../../app/src/Features/Helpers/EmailHelper.js' +import { handleAuthenticateErrors } from '../../../../../app/src/Features/Authentication/AuthenticationErrors.js' +import AuthenticationController from '../../../../../app/src/Features/Authentication/AuthenticationController.js' +import LDAPAuthenticationManager from './LDAPAuthenticationManager.mjs' + +const LDAPAuthenticationController = { + passportLogin(req, res, next) { + // This function is middleware which wraps the passport.authenticate middleware, + // so we can send back our custom `{message: {text: "", type: ""}}` responses on failure, + // and send a `{redir: ""}` response on success + passport.authenticate( + 'ldapauth', + { keepSessionInfo: true }, + async function (err, user, info, status) { + if (err) { //we cannot be here as long as errors are treated as fails + return next(err) + } + if (user) { + // `user` is either a user object or false + AuthenticationController.setAuditInfo(req, { + method: 'LDAP password login', + }) + + try { + await AuthenticationController.promises.finishLogin(user, req, res) + res.status(200) + return + } catch (err) { + return next(err) + } + } else { + if (status != 401) { + logger.warn(status, 'LDAP: ' + info.message) + } + if (EmailHelper.parseEmail(req.body.email)) return next() //Try local authentication + if (info.redir != null) { + return res.json({ redir: info.redir }) + } else { + res.status(status || info.status || 401) + delete info.status + info.type = 'error' + info.key = 'invalid-password-retry-or-reset' + const body = { message: info } + const { errorReason } = info + if (errorReason) { + body.errorReason = errorReason + delete info.errorReason + } + return res.json(body) + } + } + } + )(req, res, next) + }, + async doPassportLogin(req, profile, done) { + let user, info + try { + ;({ user, info } = await LDAPAuthenticationController._doPassportLogin( + req, + profile + )) + } catch (error) { + return done(error) + } + return done(undefined, user, info) + }, + async _doPassportLogin(req, profile) { + const { fromKnownDevice } = AuthenticationController.getAuditInfo(req) + const auditLog = { + ipAddress: req.ip, + info: { method: 'LDAP password login', fromKnownDevice }, + } + + let user, isPasswordReused + try { + user = await LDAPAuthenticationManager.promises.findOrCreateUser(profile, auditLog) + } catch (error) { + return { + user: false, + info: handleAuthenticateErrors(error, req), + } + } + if (user && AuthenticationController.captchaRequiredForLogin(req, user)) { + return { + user: false, + info: { + text: req.i18n.translate('cannot_verify_user_not_robot'), + type: 'error', + errorReason: 'cannot_verify_user_not_robot', + status: 400, + }, + } + } else if (user) { + user.externalAuth = 'ldap' + return { user, info: undefined } + } else { //we cannot be here, something is terribly wrong + logger.debug({ email : profile.mail }, 'failed LDAP log in') + return { + user: false, + info: { + type: 'error', + text: 'Unknown error', + status: 500, + }, + } + } + }, +} + +export default LDAPAuthenticationController diff --git a/services/web/modules/ldap-authentication/app/src/AuthenticationManagerLdap.mjs b/services/web/modules/authentication/ldap/app/src/LDAPAuthenticationManager.mjs similarity index 67% rename from services/web/modules/ldap-authentication/app/src/AuthenticationManagerLdap.mjs rename to services/web/modules/authentication/ldap/app/src/LDAPAuthenticationManager.mjs index 1371f76d525..66943e82a38 100644 --- a/services/web/modules/ldap-authentication/app/src/AuthenticationManagerLdap.mjs +++ b/services/web/modules/authentication/ldap/app/src/LDAPAuthenticationManager.mjs @@ -1,18 +1,13 @@ import Settings from '@overleaf/settings' import { callbackify } from '@overleaf/promise-utils' -import UserCreator from '../../../../app/src/Features/User/UserCreator.js' -import { User } from '../../../../app/src/models/User.js' +import UserCreator from '../../../../../app/src/Features/User/UserCreator.js' +import { ParallelLoginError } from '../../../../../app/src/Features/Authentication/AuthenticationErrors.js' +import { User } from '../../../../../app/src/models/User.js' +import { splitFullName } from '../../../utils.mjs' -const AuthenticationManagerLdap = { - splitFullName(fullName) { - fullName = fullName.trim(); - let lastSpaceIndex = fullName.lastIndexOf(' '); - let firstNames = fullName.substring(0, lastSpaceIndex).trim(); - let lastName = fullName.substring(lastSpaceIndex + 1).trim(); - return [firstNames, lastName]; - }, - async findOrCreateLdapUser(profile, auditLog) { - //user is already authenticated in Ldap +const LDAPAuthenticationManager = { + async findOrCreateUser(profile, auditLog) { + //user is already authenticated in LDAP const { attEmail, attFirstName, @@ -28,7 +23,7 @@ const AuthenticationManagerLdap = { : profile[attEmail].toLowerCase() let nameParts = ["",""] if ((!attFirstName || !attLastName) && attName) { - nameParts = this.splitFullName(profile[attName] || "") + nameParts = splitFullName(profile[attName] || "") } const firstName = attFirstName ? (profile[attFirstName] || "") : nameParts[0] let lastName = attLastName ? (profile[attLastName] || "") : nameParts[1] @@ -40,6 +35,7 @@ const AuthenticationManagerLdap = { profile[attAdmin] === valAdmin) } let user = await User.findOne({ 'email': email }).exec() + if( !user ) { user = await UserCreator.promises.createNewUser( { @@ -61,8 +57,12 @@ const AuthenticationManagerLdap = { userDetails.isAdmin = isAdmin } const result = await User.updateOne( - { _id: user._id, loginEpoch: user.loginEpoch }, { $inc: { loginEpoch: 1 }, $set: userDetails }, - {} + { _id: user._id, loginEpoch: user.loginEpoch }, + { + $inc: { loginEpoch: 1 }, + $set: userDetails, + $unset: { hashedPassword: "" }, + } ).exec() if (result.modifiedCount !== 1) { throw new ParallelLoginError() @@ -72,9 +72,5 @@ const AuthenticationManagerLdap = { } export default { - findOrCreateLdapUser: callbackify(AuthenticationManagerLdap.findOrCreateLdapUser), - promises: AuthenticationManagerLdap, + promises: LDAPAuthenticationManager, } -export const { - splitFullName, -} = AuthenticationManagerLdap diff --git a/services/web/modules/authentication/ldap/app/src/LDAPContacts.mjs b/services/web/modules/authentication/ldap/app/src/LDAPContacts.mjs new file mode 100644 index 00000000000..4557b4a4e41 --- /dev/null +++ b/services/web/modules/authentication/ldap/app/src/LDAPContacts.mjs @@ -0,0 +1,120 @@ +import Settings from '@overleaf/settings' +import logger from '@overleaf/logger' +import { promisify } from 'util' +import passport from 'passport' +import ldapjs from 'ldapauth-fork/node_modules/ldapjs/lib/index.js' +import UserGetter from '../../../../../app/src/Features/User/UserGetter.js' +import { splitFullName } from '../../../utils.mjs' + +function _searchLDAP(client, baseDN, options) { + return new Promise((resolve, reject) => { + const searchEntries = [] + client.search(baseDN, options, (error, res) => { + if (error) { + reject(error) + } else { + res.on('searchEntry', entry => searchEntries.push(entry.object)) + res.on('error', reject) + res.on('end', () => resolve(searchEntries)) + } + }) + }) +} + +async function fetchLDAPContacts(userId, contacts) { + if (!Settings.ldap?.enable || !process.env.OVERLEAF_LDAP_CONTACTS_FILTER) { + return [] + } + + const ldapOptions = passport._strategy('ldapauth').options.server + const { attEmail, attFirstName = "", attLastName = "", attName = "" } = Settings.ldap + const { + url, + timeout, + connectTimeout, + tlsOptions, + starttls, + bindDN, + bindCredentials + } = ldapOptions + const searchBase = process.env.OVERLEAF_LDAP_CONTACTS_SEARCH_BASE || ldapOptions.searchBase + const searchScope = process.env.OVERLEAF_LDAP_CONTACTS_SEARCH_SCOPE || 'sub' + const ldapConfig = { url, timeout, connectTimeout, tlsOptions } + + let ldapUsers + let client + + try { + await new Promise((resolve, reject) => { + client = ldapjs.createClient(ldapConfig) + client.on('error', (error) => { reject(error) }) + client.on('connectTimeout', (error) => { reject(error) }) + client.on('connect', () => { resolve() }) + }) + + if (starttls) { + const starttlsAsync = promisify(client.starttls).bind(client) + await starttlsAsync(tlsOptions, null) + } + const bindAsync = promisify(client.bind).bind(client) + await bindAsync(bindDN, bindCredentials) + + async function createContactsSearchFilter(client, ldapOptions, userId, contactsFilter) { + const searchProperty = process.env.OVERLEAF_LDAP_CONTACTS_PROPERTY + if (!searchProperty) { + return contactsFilter + } + const email = await UserGetter.promises.getUserEmail(userId) + const searchOptions = { + scope: ldapOptions.searchScope, + attributes: [searchProperty], + filter: `(${Settings.ldap.attEmail}=${email})` + } + const searchBase = ldapOptions.searchBase + const ldapUser = (await _searchLDAP(client, searchBase, searchOptions))[0] + const searchPropertyValue = ldapUser ? ldapUser[searchProperty] + : process.env.OVERLEAF_LDAP_CONTACTS_NON_LDAP_VALUE || 'IMATCHNOTHING' + return contactsFilter.replace(/{{userProperty}}/g, searchPropertyValue) + } + + const filter = await createContactsSearchFilter(client, ldapOptions, userId, process.env.OVERLEAF_LDAP_CONTACTS_FILTER) + const searchOptions = { scope: searchScope, attributes: [attEmail, attFirstName, attLastName, attName], filter } + + ldapUsers = await _searchLDAP(client, searchBase, searchOptions) + } catch (error) { + logger.warn({ error }, 'Error in fetchLDAPContacts') + return [] + } finally { + client?.unbind() + } + + const newLDAPContacts = ldapUsers.reduce((acc, ldapUser) => { + const email = Array.isArray(ldapUser[attEmail]) + ? ldapUser[attEmail][0]?.toLowerCase() + : ldapUser[attEmail]?.toLowerCase() + if (!email) return acc + if (!contacts.some(contact => contact.email === email)) { + let nameParts = ["", ""] + if ((!attFirstName || !attLastName) && attName) { + nameParts = splitFullName(ldapUser[attName] || "") + } + const firstName = attFirstName ? (ldapUser[attFirstName] || "") : nameParts[0] + const lastName = attLastName ? (ldapUser[attLastName] || "") : nameParts[1] + acc.push({ + first_name: firstName, + last_name: lastName, + email: email, + type: 'user' + }) + } + return acc + }, []) + + return newLDAPContacts.sort((a, b) => + a.last_name.localeCompare(b.last_name) || + a.first_name.localeCompare(b.first_name) || + a.email.localeCompare(b.email) + ) +} + +export default fetchLDAPContacts diff --git a/services/web/modules/authentication/ldap/app/src/LDAPModuleManager.mjs b/services/web/modules/authentication/ldap/app/src/LDAPModuleManager.mjs new file mode 100644 index 00000000000..846ca9b1583 --- /dev/null +++ b/services/web/modules/authentication/ldap/app/src/LDAPModuleManager.mjs @@ -0,0 +1,112 @@ +import logger from '@overleaf/logger' +import passport from 'passport' +import { Strategy as LDAPStrategy } from 'passport-ldapauth' +import Settings from '@overleaf/settings' +import PermissionsManager from '../../../../../app/src/Features/Authorization/PermissionsManager.js' +import { readFilesContentFromEnv, numFromEnv, boolFromEnv } from '../../../utils.mjs' +import LDAPAuthenticationController from './LDAPAuthenticationController.mjs' +import fetchLDAPContacts from './LDAPContacts.mjs' + +const LDAPModuleManager = { + initSettings() { + Settings.ldap = { + enable: true, + placeholder: process.env.OVERLEAF_LDAP_PLACEHOLDER || 'Username', + attEmail: process.env.OVERLEAF_LDAP_EMAIL_ATT || 'mail', + attFirstName: process.env.OVERLEAF_LDAP_FIRST_NAME_ATT, + attLastName: process.env.OVERLEAF_LDAP_LAST_NAME_ATT, + attName: process.env.OVERLEAF_LDAP_NAME_ATT, + attAdmin: process.env.OVERLEAF_LDAP_IS_ADMIN_ATT, + valAdmin: process.env.OVERLEAF_LDAP_IS_ADMIN_ATT_VALUE, + updateUserDetailsOnLogin: boolFromEnv(process.env.OVERLEAF_LDAP_UPDATE_USER_DETAILS_ON_LOGIN), + } + }, + passportSetup(passport, callback) { + const ldapOptions = { + url: process.env.OVERLEAF_LDAP_URL, + bindDN: process.env.OVERLEAF_LDAP_BIND_DN || "", + bindCredentials: process.env.OVERLEAF_LDAP_BIND_CREDENTIALS || "", + bindProperty: process.env.OVERLEAF_LDAP_BIND_PROPERTY, + searchBase: process.env.OVERLEAF_LDAP_SEARCH_BASE, + searchFilter: process.env.OVERLEAF_LDAP_SEARCH_FILTER, + searchScope: process.env.OVERLEAF_LDAP_SEARCH_SCOPE || 'sub', + searchAttributes: JSON.parse(process.env.OVERLEAF_LDAP_SEARCH_ATTRIBUTES || '[]'), + groupSearchBase: process.env.OVERLEAF_LDAP_ADMIN_SEARCH_BASE, + groupSearchFilter: process.env.OVERLEAF_LDAP_ADMIN_SEARCH_FILTER, + groupSearchScope: process.env.OVERLEAF_LDAP_ADMIN_SEARCH_SCOPE || 'sub', + groupSearchAttributes: ["dn"], + groupDnProperty: process.env.OVERLEAF_LDAP_ADMIN_DN_PROPERTY, + cache: boolFromEnv(process.env.OVERLEAF_LDAP_CACHE), + timeout: numFromEnv(process.env.OVERLEAF_LDAP_TIMEOUT), + connectTimeout: numFromEnv(process.env.OVERLEAF_LDAP_CONNECT_TIMEOUT), + starttls: boolFromEnv(process.env.OVERLEAF_LDAP_STARTTLS), + tlsOptions: { + ca: readFilesContentFromEnv(process.env.OVERLEAF_LDAP_TLS_OPTS_CA_PATH), + rejectUnauthorized: boolFromEnv(process.env.OVERLEAF_LDAP_TLS_OPTS_REJECT_UNAUTH), + } + } + try { + passport.use( + new LDAPStrategy( + { + server: ldapOptions, + passReqToCallback: true, + usernameField: 'email', + passwordField: 'password', + handleErrorsAsFailures: true, + }, + LDAPAuthenticationController.doPassportLogin + ) + ) + callback(null) + } catch (error) { + callback(error) + } + }, + + async getContacts(userId, contacts, callback) { + try { + const newContacts = await fetchLDAPContacts(userId, contacts) + callback(null, newContacts) + } catch (error) { + callback(error) + } + }, + + initPolicy() { + try { + PermissionsManager.registerCapability('change-password', { default : true }) + } catch (error) { + logger.info({}, error.message) + } + const ldapPolicyValidator = async ({ user, subscription }) => { +// If user is not logged in, user.externalAuth is undefined, +// in this case allow to change password if the user has a hashedPassword + return user.externalAuth === 'ldap' || (user.externalAuth === undefined && !user.hashedPassword) + } + try { + PermissionsManager.registerPolicy( + 'ldapPolicy', + { 'change-password' : false }, + { validator: ldapPolicyValidator } + ) + } catch (error) { + logger.info({}, error.message) + } + }, + async getGroupPolicyForUser(user, callback) { + try { + const userValidationMap = await PermissionsManager.promises.getUserValidationStatus({ + user, + groupPolicy : { 'ldapPolicy' : true }, + subscription : null + }) + let groupPolicy = Object.fromEntries(userValidationMap) + callback(null, {'groupPolicy' : groupPolicy }) + } catch (error) { + callback(error) + } + }, +} + +export default LDAPModuleManager diff --git a/services/web/modules/authentication/ldap/app/src/LDAPRouter.mjs b/services/web/modules/authentication/ldap/app/src/LDAPRouter.mjs new file mode 100644 index 00000000000..44d9d373d2d --- /dev/null +++ b/services/web/modules/authentication/ldap/app/src/LDAPRouter.mjs @@ -0,0 +1,19 @@ +import logger from '@overleaf/logger' +import RateLimiterMiddleware from '../../../../../app/src/Features/Security/RateLimiterMiddleware.js' +import CaptchaMiddleware from '../../../../../app/src/Features/Captcha/CaptchaMiddleware.js' +import AuthenticationController from '../../../../../app/src/Features/Authentication/AuthenticationController.js' +import { overleafLoginRateLimiter } from '../../../../../app/src/infrastructure/RateLimiter.js' +import LDAPAuthenticationController from './LDAPAuthenticationController.mjs' + +export default { + apply(webRouter) { + logger.debug({}, 'Init LDAP router') + webRouter.post('/login', + RateLimiterMiddleware.rateLimit(overleafLoginRateLimiter), // rate limit IP (20 / 60s) + RateLimiterMiddleware.loginRateLimitEmail, // rate limit email (10 / 120s) + CaptchaMiddleware.validateCaptcha('login'), + LDAPAuthenticationController.passportLogin, + AuthenticationController.passportLogin, + ) + }, +} diff --git a/services/web/modules/authentication/ldap/index.mjs b/services/web/modules/authentication/ldap/index.mjs new file mode 100644 index 00000000000..244a8db8e72 --- /dev/null +++ b/services/web/modules/authentication/ldap/index.mjs @@ -0,0 +1,17 @@ +let ldapModule = {} +if (process.env.EXTERNAL_AUTH.includes('ldap')) { + const { default: LDAPModuleManager } = await import('./app/src/LDAPModuleManager.mjs') + const { default: router } = await import('./app/src/LDAPRouter.mjs') + LDAPModuleManager.initSettings() + LDAPModuleManager.initPolicy() + ldapModule = { + name: 'ldap-authentication', + hooks: { + passportSetup: LDAPModuleManager.passportSetup, + getContacts: LDAPModuleManager.getContacts, + getGroupPolicyForUser: LDAPModuleManager.getGroupPolicyForUser, + }, + router: router, + } +} +export default ldapModule diff --git a/services/web/modules/authentication/logout.mjs b/services/web/modules/authentication/logout.mjs new file mode 100644 index 00000000000..4163cf536d5 --- /dev/null +++ b/services/web/modules/authentication/logout.mjs @@ -0,0 +1,18 @@ +let SAMLAuthenticationController +if (process.env.EXTERNAL_AUTH.includes('saml')) { + SAMLAuthenticationController = await import('./saml/app/src/SAMLAuthenticationController.mjs') +} +let OIDCAuthenticationController +if (process.env.EXTERNAL_AUTH.includes('oidc')) { + OIDCAuthenticationController = await import('./oidc/app/src/OIDCAuthenticationController.mjs') +} +export default async function logout(req, res, next) { + switch(req.user.externalAuth) { + case 'saml': + return SAMLAuthenticationController.default.passportLogout(req, res, next) + case 'oidc': + return OIDCAuthenticationController.default.passportLogout(req, res, next) + default: + next() + } +} diff --git a/services/web/modules/authentication/oidc/app/src/OIDCAuthenticationController.mjs b/services/web/modules/authentication/oidc/app/src/OIDCAuthenticationController.mjs new file mode 100644 index 00000000000..42c01e712f9 --- /dev/null +++ b/services/web/modules/authentication/oidc/app/src/OIDCAuthenticationController.mjs @@ -0,0 +1,171 @@ +import logger from '@overleaf/logger' +import passport from 'passport' +import Settings from '@overleaf/settings' +import AuthenticationController from '../../../../../app/src/Features/Authentication/AuthenticationController.js' +import UserController from '../../../../../app/src/Features/User/UserController.js' +import ThirdPartyIdentityManager from '../../../../../app/src/Features/User/ThirdPartyIdentityManager.js' +import OIDCAuthenticationManager from './OIDCAuthenticationManager.mjs' +import { acceptsJson } from '../../../../../app/src/infrastructure/RequestContentTypeDetection.js' + +const OIDCAuthenticationController = { + passportLogin(req, res, next) { + req.session.intent = req.query.intent + passport.authenticate('openidconnect')(req, res, next) + }, + passportLoginCallback(req, res, next) { + passport.authenticate( + 'openidconnect', + { keepSessionInfo: true }, + async function (err, user, info) { + if (err) { + return next(err) + } + if(req.session.intent === 'link') { + delete req.session.intent +// After linking, log out from the OIDC provider and redirect back to '/user/settings'. +// Keycloak supports this; Authentik does not (yet). + const logoutUrl = process.env.OVERLEAF_OIDC_LOGOUT_URL + const redirectUri = `${Settings.siteUrl.replace(/\/+$/, '')}/user/settings` + return res.redirect(`${logoutUrl}?id_token_hint=${info.idToken}&post_logout_redirect_uri=${encodeURIComponent(redirectUri)}`) + } + if (user) { + req.session.idToken = info.idToken + user.externalAuth = 'oidc' + // `user` is either a user object or false + AuthenticationController.setAuditInfo(req, { + method: 'OIDC login', + }) + try { + await AuthenticationController.promises.finishLogin(user, req, res) + } catch (err) { + return next(err) + } + } else { + if (info.redir != null) { + return res.json({ redir: info.redir }) + } else { + res.status(info.status || 401) + delete info.status + const body = { message: info } + return res.json(body) + } + } + } + )(req, res, next) + }, + async doPassportLogin(req, issuer, profile, context, idToken, accessToken, refreshToken, done) { + let user, info + try { + if(req.session.intent === 'link') { + ;({ user, info } = await OIDCAuthenticationController._doLink( + req, + profile + )) + } else { + ;({ user, info } = await OIDCAuthenticationController._doLogin( + req, + profile + )) + } + } catch (error) { + return done(error) + } + if (user) { + info = { + ...(info || {}), + idToken: idToken + } + } + return done(null, user, info) + }, + async _doLogin(req, profile) { + const { fromKnownDevice } = AuthenticationController.getAuditInfo(req) + const auditLog = { + ipAddress: req.ip, + info: { method: 'OIDC login', fromKnownDevice }, + } + + let user + try { + user = await OIDCAuthenticationManager.promises.findOrCreateUser(profile, auditLog) + } catch (error) { + logger.debug({ email : profile.emails[0].value }, `OIDC login failed: ${error}`) + return { + user: false, + info: { + type: 'error', + text: error.message, + status: 401, + }, + } + } + if (user) { + return { user, info: undefined } + } else { // we cannot be here, something is terribly wrong + logger.debug({ email : profile.emails[0].value }, 'failed OIDC log in') + return { + user: false, + info: { + type: 'error', + text: 'Unknown error', + status: 500, + }, + } + } + }, + async _doLink(req, profile) { + const { user: { _id: userId }, ip } = req + try { + const auditLog = { + ipAddress: ip, + initiatorId: userId, + } + await OIDCAuthenticationManager.promises.linkAccount(userId, profile, auditLog) + } catch (error) { + logger.error(error.info, error.message) + return { + user: true, + info: { + type: 'error', + text: error.message, + status: 200, + }, + } + } + return { user: true, info: undefined } + }, + async unlinkAccount(req, res, next) { + try { + const { user: { _id: userId }, body: { providerId }, ip } = req + const auditLog = { + ipAddress: ip, + initiatorId: userId, + } + await ThirdPartyIdentityManager.promises.unlink(userId, providerId, auditLog) + return res.status(200).end() + } catch (error) { + logger.error(error.info, error.message) + return { + user: false, + info: { + type: 'error', + text: 'Can not unlink account', + status: 200, + } + } + } + }, + async passportLogout(req, res, next) { +// TODO: instead of storing idToken in session, use refreshToken to obtain a new idToken? + const idTokenHint = req.session.idToken + await UserController.promises.doLogout(req) + const logoutUrl = process.env.OVERLEAF_OIDC_LOGOUT_URL + const redirectUri = Settings.siteUrl + res.redirect(`${logoutUrl}?id_token_hint=${idTokenHint}&post_logout_redirect_uri=${encodeURIComponent(redirectUri)}`) + }, + passportLogoutCallback(req, res, next) { + const redirectUri = Settings.siteUrl + res.redirect(redirectUri) + }, +} +export default OIDCAuthenticationController diff --git a/services/web/modules/authentication/oidc/app/src/OIDCAuthenticationManager.mjs b/services/web/modules/authentication/oidc/app/src/OIDCAuthenticationManager.mjs new file mode 100644 index 00000000000..56ec2e5455d --- /dev/null +++ b/services/web/modules/authentication/oidc/app/src/OIDCAuthenticationManager.mjs @@ -0,0 +1,94 @@ +import Settings from '@overleaf/settings' +import UserCreator from '../../../../../app/src/Features/User/UserCreator.js' +import ThirdPartyIdentityManager from '../../../../../app/src/Features/User/ThirdPartyIdentityManager.js' +import { ParallelLoginError } from '../../../../../app/src/Features/Authentication/AuthenticationErrors.js' +import { User } from '../../../../../app/src/models/User.js' + +const OIDCAuthenticationManager = { + async findOrCreateUser(profile, auditLog) { + const { + attUserId, + attAdmin, + valAdmin, + updateUserDetailsOnLogin, + providerId, + } = Settings.oidc + const oidcUserId = profile[attUserId] + const email = profile.emails[0].value + const firstName = profile.name?.givenName || "" + const lastName = profile.name?.familyName || "" + let isAdmin = false + if (attAdmin && valAdmin) { + if (attAdmin === 'email') { + isAdmin = (email === valAdmin) + } else { + isAdmin = (profile[attAdmin] === valAdmin) + } + } + const oidcUserData = null // Possibly it can be used later + let user + try { + user = await ThirdPartyIdentityManager.promises.login(providerId, oidcUserId, oidcUserData) + } catch { +// A user with the specified OIDC ID and provider ID is not found. Search for a user with the given email. +// If no user exists with this email, create a new user and link the OIDC account to it. +// If a user exists but no account from the specified OIDC provider is linked to this user, link the OIDC account to this user. +// If an account from the specified provider is already linked to this user, unlink it, and link the OIDC account to this user. +// (Is it safe? Concider: If an account from the specified provider is already linked to this user, throw an error) + user = await User.findOne({ 'email': email }).exec() + if (!user) { + user = await UserCreator.promises.createNewUser( + { + email: email, + first_name: firstName, + last_name: lastName, + isAdmin: isAdmin, + holdingAccount: false, + } + ) + } +// const alreadyLinked = user.thirdPartyIdentifiers.some(item => item.providerId === providerId) +// if (!alreadyLinked) { + auditLog.initiatorId = user._id + await ThirdPartyIdentityManager.promises.link(user._id, providerId, oidcUserId, oidcUserData, auditLog) + await User.updateOne( + { _id: user._id }, + { $set : { + 'emails.0.confirmedAt': Date.now(), //email of external user is confirmed + }, + } + ).exec() +// } else { +// throw new Error(`Overleaf user ${user.email} is already linked to another ${providerId} user`) +// } + } + + let userDetails = updateUserDetailsOnLogin ? { first_name : firstName, last_name: lastName } : {} + if (attAdmin && valAdmin) { + user.isAdmin = isAdmin + userDetails.isAdmin = isAdmin + } + const result = await User.updateOne( + { _id: user._id, loginEpoch: user.loginEpoch }, { $inc: { loginEpoch: 1 }, $set: userDetails }, + {} + ).exec() + + if (result.modifiedCount !== 1) { + throw new ParallelLoginError() + } + return user + }, + async linkAccount(userId, profile, auditLog) { + const { + attUserId, + providerId, + } = Settings.oidc + const oidcUserId = profile[attUserId] + const oidcUserData = null // Possibly it can be used later + await ThirdPartyIdentityManager.promises.link(userId, providerId, oidcUserId, oidcUserData, auditLog) + }, +} + +export default { + promises: OIDCAuthenticationManager, +} diff --git a/services/web/modules/authentication/oidc/app/src/OIDCModuleManager.mjs b/services/web/modules/authentication/oidc/app/src/OIDCModuleManager.mjs new file mode 100644 index 00000000000..3a2e6e27802 --- /dev/null +++ b/services/web/modules/authentication/oidc/app/src/OIDCModuleManager.mjs @@ -0,0 +1,82 @@ +import logger from '@overleaf/logger' +import passport from 'passport' +import Settings from '@overleaf/settings' +import { readFilesContentFromEnv, numFromEnv, boolFromEnv } from '../../../utils.mjs' +import PermissionsManager from '../../../../../app/src/Features/Authorization/PermissionsManager.js' +import OIDCAuthenticationController from './OIDCAuthenticationController.mjs' +import { Strategy as OIDCStrategy } from 'passport-openidconnect' + +const OIDCModuleManager = { + initSettings() { + let providerId = process.env.OVERLEAF_OIDC_PROVIDER_ID || 'oidc' + Settings.oidc = { + enable: true, + providerId: providerId, + identityServiceName: process.env.OVERLEAF_OIDC_IDENTITY_SERVICE_NAME || `Log in with ${Settings.oauthProviders[providerId].name}`, + attUserId: process.env.OVERLEAF_OIDC_USER_ID_FIELD || 'id', + attAdmin: process.env.OVERLEAF_OIDC_IS_ADMIN_FIELD, + valAdmin: process.env.OVERLEAF_OIDC_IS_ADMIN_FIELD_VALUE, + updateUserDetailsOnLogin: boolFromEnv(process.env.OVERLEAF_OIDC_UPDATE_USER_DETAILS_ON_LOGIN), + } + }, + passportSetup(passport, callback) { + const oidcOptions = { + issuer: process.env.OVERLEAF_OIDC_ISSUER, + authorizationURL: process.env.OVERLEAF_OIDC_AUTHORIZATION_URL, + tokenURL: process.env.OVERLEAF_OIDC_TOKEN_URL, + userInfoURL: process.env.OVERLEAF_OIDC_USER_INFO_URL, + clientID: process.env.OVERLEAF_OIDC_CLIENT_ID, + clientSecret: process.env.OVERLEAF_OIDC_CLIENT_SECRET, + callbackURL: `${Settings.siteUrl.replace(/\/+$/, '')}/oidc/login/callback`, + scope: process.env.OVERLEAF_OIDC_SCOPE || 'openid profile email', + passReqToCallback: true, + } + try { + passport.use( + new OIDCStrategy( + oidcOptions, + OIDCAuthenticationController.doPassportLogin + ) + ) + callback(null) + } catch (error) { + callback(error) + } + }, + initPolicy() { + try { + PermissionsManager.registerCapability('change-password', { default : true }) + } catch (error) { + logger.info({}, error.message) + } + const oidcPolicyValidator = async ({ user, subscription }) => { +// If user is not logged in, user.externalAuth is undefined, +// in this case allow to change password if the user has a hashedPassword + return user.externalAuth === 'oidc' || (user.externalAuth === undefined && !user.hashedPassword) + } + try { + PermissionsManager.registerPolicy( + 'oidcPolicy', + { 'change-password' : false }, + { validator: oidcPolicyValidator } + ) + } catch (error) { + logger.info({}, error.message) + } + }, + async getGroupPolicyForUser(user, callback) { + try { + const userValidationMap = await PermissionsManager.promises.getUserValidationStatus({ + user, + groupPolicy : { 'oidcPolicy' : true }, + subscription : null + }) + let groupPolicy = Object.fromEntries(userValidationMap) + callback(null, {'groupPolicy' : groupPolicy }) + } catch (error) { + callback(error) + } + }, +} + +export default OIDCModuleManager diff --git a/services/web/modules/authentication/oidc/app/src/OIDCRouter.mjs b/services/web/modules/authentication/oidc/app/src/OIDCRouter.mjs new file mode 100644 index 00000000000..519fa5043a8 --- /dev/null +++ b/services/web/modules/authentication/oidc/app/src/OIDCRouter.mjs @@ -0,0 +1,15 @@ +import logger from '@overleaf/logger' +import UserController from '../../../../../app/src/Features/User/UserController.js' +import OIDCAuthenticationController from './OIDCAuthenticationController.mjs' +import logout from '../../../logout.mjs' + +export default { + apply(webRouter) { + logger.debug({}, 'Init OIDC router') + webRouter.get('/oidc/login', OIDCAuthenticationController.passportLogin) + webRouter.get('/oidc/login/callback', OIDCAuthenticationController.passportLoginCallback) + webRouter.get('/oidc/logout/callback', OIDCAuthenticationController.passportLogoutCallback) + webRouter.post('/user/oauth-unlink', OIDCAuthenticationController.unlinkAccount) + webRouter.post('/logout', logout, UserController.logout) + }, +} diff --git a/services/web/modules/authentication/oidc/index.mjs b/services/web/modules/authentication/oidc/index.mjs new file mode 100644 index 00000000000..51d9e0d483d --- /dev/null +++ b/services/web/modules/authentication/oidc/index.mjs @@ -0,0 +1,16 @@ +let oidcModule = {} +if (process.env.EXTERNAL_AUTH.includes('oidc')) { + const { default: OIDCModuleManager } = await import('./app/src/OIDCModuleManager.mjs') + const { default: router } = await import('./app/src/OIDCRouter.mjs') + OIDCModuleManager.initSettings() + OIDCModuleManager.initPolicy() + oidcModule = { + name: 'oidc-authentication', + hooks: { + passportSetup: OIDCModuleManager.passportSetup, + getGroupPolicyForUser: OIDCModuleManager.getGroupPolicyForUser, + }, + router: router, + } +} +export default oidcModule diff --git a/services/web/modules/saml-authentication/app/src/AuthenticationControllerSaml.mjs b/services/web/modules/authentication/saml/app/src/SAMLAuthenticationController.mjs similarity index 65% rename from services/web/modules/saml-authentication/app/src/AuthenticationControllerSaml.mjs rename to services/web/modules/authentication/saml/app/src/SAMLAuthenticationController.mjs index f5db3f738dd..ac0e5398b28 100644 --- a/services/web/modules/saml-authentication/app/src/AuthenticationControllerSaml.mjs +++ b/services/web/modules/authentication/saml/app/src/SAMLAuthenticationController.mjs @@ -1,15 +1,16 @@ import Settings from '@overleaf/settings' import logger from '@overleaf/logger' import passport from 'passport' -import AuthenticationController from '../../../../app/src/Features/Authentication/AuthenticationController.js' -import AuthenticationManagerSaml from './AuthenticationManagerSaml.mjs' -import UserController from '../../../../app/src/Features/User/UserController.js' -import UserSessionsManager from '../../../../app/src/Features/User/UserSessionsManager.js' -import { handleAuthenticateErrors } from '../../../../app/src/Features/Authentication/AuthenticationErrors.js' -import { xmlResponse } from '../../../../app/src/infrastructure/Response.js' +import AuthenticationController from '../../../../../app/src/Features/Authentication/AuthenticationController.js' +import SAMLAuthenticationManager from './SAMLAuthenticationManager.mjs' +import UserController from '../../../../../app/src/Features/User/UserController.js' +import UserSessionsManager from '../../../../../app/src/Features/User/UserSessionsManager.js' +import { handleAuthenticateErrors } from '../../../../../app/src/Features/Authentication/AuthenticationErrors.js' +import { xmlResponse } from '../../../../../app/src/infrastructure/Response.js' +import { readFilesContentFromEnv } from '../../../utils.mjs' -const AuthenticationControllerSaml = { - passportSamlAuthWithIdP(req, res, next) { +const SAMLAuthenticationController = { + passportLogin(req, res, next) { if ( passport._strategy('saml')._saml.options.authnRequestBinding === 'HTTP-POST') { const csp = res.getHeader('Content-Security-Policy') if (csp) { @@ -21,7 +22,7 @@ const AuthenticationControllerSaml = { } passport.authenticate('saml')(req, res, next) }, - passportSamlLogin(req, res, next) { + passportLoginCallback(req, res, next) { // This function is middleware which wraps the passport.authenticate middleware, // so we can send back our custom `{message: {text: "", type: ""}}` responses on failure, // and send a `{redir: ""}` response on success @@ -46,24 +47,19 @@ const AuthenticationControllerSaml = { if (info.redir != null) { return res.json({ redir: info.redir }) } else { - res.status(info.status || 200) + res.status(info.status || 401) delete info.status const body = { message: info } - const { errorReason } = info - if (errorReason) { - body.errorReason = errorReason - delete info.errorReason - } return res.json(body) } } } )(req, res, next) }, - async doPassportSamlLogin(req, profile, done) { + async doPassportLogin(req, profile, done) { let user, info try { - ;({ user, info } = await AuthenticationControllerSaml._doPassportSamlLogin( + ;({ user, info } = await SAMLAuthenticationController._doPassportLogin( req, profile )) @@ -72,7 +68,7 @@ const AuthenticationControllerSaml = { } return done(undefined, user, info) }, - async _doPassportSamlLogin(req, profile) { + async _doPassportLogin(req, profile) { const { fromKnownDevice } = AuthenticationController.getAuditInfo(req) const auditLog = { ipAddress: req.ip, @@ -81,7 +77,7 @@ const AuthenticationControllerSaml = { let user try { - user = await AuthenticationManagerSaml.promises.findOrCreateSamlUser(profile, auditLog) + user = await SAMLAuthenticationManager.promises.findOrCreateUser(profile, auditLog) } catch (error) { return { user: false, @@ -89,9 +85,10 @@ const AuthenticationControllerSaml = { } } if (user) { + user.externalAuth = 'saml' req.session.saml_extce = {nameID : profile.nameID, sessionIndex : profile.sessionIndex} return { user, info: undefined } - } else { //something wrong + } else { // we cannot be here, something is terribly wrong logger.debug({ email : profile.mail }, 'failed SAML log in') return { user: false, @@ -103,23 +100,24 @@ const AuthenticationControllerSaml = { } } }, - async passportSamlSPLogout(req, res, next) { + async passportLogout(req, res, next) { passport._strategy('saml').logout(req, async (err, url) => { - if (err) logger.error({ err }, 'can not generate logout url') await UserController.promises.doLogout(req) + if (err) return next(err) res.redirect(url) }) }, - passportSamlIdPLogout(req, res, next) { + passportLogoutCallback(req, res, next) { +//TODO: is it possible to close the editor? passport.authenticate('saml')(req, res, (err) => { if (err) return next(err) res.redirect('/login'); }) }, - async doPassportSamlLogout(req, profile, done) { + async doPassportLogout(req, profile, done) { let user, info try { - ;({ user, info } = await AuthenticationControllerSaml._doPassportSamlLogout( + ;({ user, info } = await SAMLAuthenticationController._doPassportLogout( req, profile )) @@ -128,7 +126,7 @@ const AuthenticationControllerSaml = { } return done(undefined, user, info) }, - async _doPassportSamlLogout(req, profile) { + async _doPassportLogout(req, profile) { if (req?.session?.saml_extce?.nameID === profile.nameID && req?.session?.saml_extce?.sessionIndex === profile.sessionIndex) { profile = req.user @@ -138,23 +136,15 @@ const AuthenticationControllerSaml = { }) return { user: profile, info: undefined } }, - passportSamlMetadata(req, res) { + getSPMetadata(req, res) { const samlStratery = passport._strategy('saml') res.setHeader('Content-Disposition', `attachment; filename="${samlStratery._saml.options.issuer}-meta.xml"`) xmlResponse(res, samlStratery.generateServiceProviderMetadata( - samlStratery._saml.options.decryptionCert, - samlStratery._saml.options.signingCert + readFilesContentFromEnv(process.env.OVERLEAF_SAML_DECRYPTION_CERT), + readFilesContentFromEnv(process.env.OVERLEAF_SAML_PUBLIC_CERT) ) ) }, } -export const { - passportSamlAuthWithIdP, - passportSamlLogin, - passportSamlSPLogout, - passportSamlIdPLogout, - doPassportSamlLogin, - doPassportSamlLogout, - passportSamlMetadata, -} = AuthenticationControllerSaml +export default SAMLAuthenticationController diff --git a/services/web/modules/authentication/saml/app/src/SAMLAuthenticationManager.mjs b/services/web/modules/authentication/saml/app/src/SAMLAuthenticationManager.mjs new file mode 100644 index 00000000000..80c4e30ea7a --- /dev/null +++ b/services/web/modules/authentication/saml/app/src/SAMLAuthenticationManager.mjs @@ -0,0 +1,85 @@ +import Settings from '@overleaf/settings' +import UserCreator from '../../../../../app/src/Features/User/UserCreator.js' +import { ParallelLoginError } from '../../../../../app/src/Features/Authentication/AuthenticationErrors.js' +import SAMLIdentityManager from '../../../../../app/src/Features/User/SAMLIdentityManager.js' +import { User } from '../../../../../app/src/models/User.js' + +const SAMLAuthenticationManager = { + async findOrCreateUser(profile, auditLog) { + const { + attUserId, + attEmail, + attFirstName, + attLastName, + attAdmin, + valAdmin, + updateUserDetailsOnLogin, + } = Settings.saml + const externalUserId = profile[attUserId] + const email = Array.isArray(profile[attEmail]) + ? profile[attEmail][0].toLowerCase() + : profile[attEmail].toLowerCase() + const firstName = attFirstName ? profile[attFirstName] : "" + const lastName = attLastName ? profile[attLastName] : email + let isAdmin = false + if (attAdmin && valAdmin) { + isAdmin = (Array.isArray(profile[attAdmin]) ? profile[attAdmin].includes(valAdmin) : + profile[attAdmin] === valAdmin) + } + const providerId = '1' // for now, only one fixed IdP is supported +// We search for a SAML user, and if none is found, we search for a user with the given email. If a user is found, +// we update the user to be a SAML user, otherwise, we create a new SAML user with the given email. In the case of +// multiple SAML IdPs, one would have to do something similar, or possibly report an error like +// 'the email is associated with the wrong IdP' + let user = await SAMLIdentityManager.getUser(providerId, externalUserId, attUserId) + if (!user) { + user = await User.findOne({ 'email': email }).exec() + if (!user) { + user = await UserCreator.promises.createNewUser( + { + email: email, + first_name: firstName, + last_name: lastName, + isAdmin: isAdmin, + holdingAccount: false, + samlIdentifiers: [{ providerId: providerId }], + } + ) + } + // cannot use SAMLIdentityManager.linkAccounts because affilations service is not there + await User.updateOne( + { _id: user._id }, + { + $set : { + 'emails.0.confirmedAt': Date.now(), //email of saml user is confirmed + 'emails.0.samlProviderId': providerId, + 'samlIdentifiers.0.providerId': providerId, + 'samlIdentifiers.0.externalUserId': externalUserId, + 'samlIdentifiers.0.userIdAttribute': attUserId, + }, + } + ).exec() + } + let userDetails = updateUserDetailsOnLogin ? { first_name : firstName, last_name: lastName } : {} + if (attAdmin && valAdmin) { + user.isAdmin = isAdmin + userDetails.isAdmin = isAdmin + } + const result = await User.updateOne( + { _id: user._id, loginEpoch: user.loginEpoch }, + { + $inc: { loginEpoch: 1 }, + $set: userDetails, + $unset: { hashedPassword: "" }, + }, + ).exec() + if (result.modifiedCount !== 1) { + throw new ParallelLoginError() + } + return user + }, +} + +export default { + promises: SAMLAuthenticationManager, +} diff --git a/services/web/modules/authentication/saml/app/src/SAMLModuleManager.mjs b/services/web/modules/authentication/saml/app/src/SAMLModuleManager.mjs new file mode 100644 index 00000000000..c7efdef214c --- /dev/null +++ b/services/web/modules/authentication/saml/app/src/SAMLModuleManager.mjs @@ -0,0 +1,100 @@ +import logger from '@overleaf/logger' +import passport from 'passport' +import Settings from '@overleaf/settings' +import { readFilesContentFromEnv, numFromEnv, boolFromEnv } from '../../../utils.mjs' +import PermissionsManager from '../../../../../app/src/Features/Authorization/PermissionsManager.js' +import SAMLAuthenticationController from './SAMLAuthenticationController.mjs' +import { Strategy as SAMLStrategy } from '@node-saml/passport-saml' + +const SAMLModuleManager = { + initSettings() { + Settings.saml = { + enable: true, + identityServiceName: process.env.OVERLEAF_SAML_IDENTITY_SERVICE_NAME || 'Log in with SAML IdP', + attUserId: process.env.OVERLEAF_SAML_USER_ID_FIELD || 'nameID', + attEmail: process.env.OVERLEAF_SAML_EMAIL_FIELD || 'nameID', + attFirstName: process.env.OVERLEAF_SAML_FIRST_NAME_FIELD || 'givenName', + attLastName: process.env.OVERLEAF_SAML_LAST_NAME_FIELD || 'lastName', + attAdmin: process.env.OVERLEAF_SAML_IS_ADMIN_FIELD, + valAdmin: process.env.OVERLEAF_SAML_IS_ADMIN_FIELD_VALUE, + updateUserDetailsOnLogin: boolFromEnv(process.env.OVERLEAF_SAML_UPDATE_USER_DETAILS_ON_LOGIN), + } +}, + passportSetup(passport, callback) { + const samlOptions = { + entryPoint: process.env.OVERLEAF_SAML_ENTRYPOINT, + callbackUrl: `${Settings.siteUrl.replace(/\/+$/, '')}/saml/login/callback`, + issuer: process.env.OVERLEAF_SAML_ISSUER, + audience: process.env.OVERLEAF_SAML_AUDIENCE, + cert: readFilesContentFromEnv(process.env.OVERLEAF_SAML_IDP_CERT), + privateKey: readFilesContentFromEnv(process.env.OVERLEAF_SAML_PRIVATE_KEY), + decryptionPvk: readFilesContentFromEnv(process.env.OVERLEAF_SAML_DECRYPTION_PVK), + signatureAlgorithm: process.env.OVERLEAF_SAML_SIGNATURE_ALGORITHM, + additionalParams: JSON.parse(process.env.OVERLEAF_SAML_ADDITIONAL_PARAMS || '{}'), + additionalAuthorizeParams: JSON.parse(process.env.OVERLEAF_SAML_ADDITIONAL_AUTHORIZE_PARAMS || '{}'), + identifierFormat: process.env.OVERLEAF_SAML_IDENTIFIER_FORMAT, + acceptedClockSkewMs: numFromEnv(process.env.OVERLEAF_SAML_ACCEPTED_CLOCK_SKEW_MS), + attributeConsumingServiceIndex: process.env.OVERLEAF_SAML_ATTRIBUTE_CONSUMING_SERVICE_INDEX, + authnContext: process.env.OVERLEAF_SAML_AUTHN_CONTEXT ? JSON.parse(process.env.OVERLEAF_SAML_AUTHN_CONTEXT) : undefined, + forceAuthn: boolFromEnv(process.env.OVERLEAF_SAML_FORCE_AUTHN), + disableRequestedAuthnContext: boolFromEnv(process.env.OVERLEAF_SAML_DISABLE_REQUESTED_AUTHN_CONTEXT), + skipRequestCompression: process.env.OVERLEAF_SAML_AUTHN_REQUEST_BINDING === 'HTTP-POST', // compression should be skipped iff authnRequestBinding is POST + authnRequestBinding: process.env.OVERLEAF_SAML_AUTHN_REQUEST_BINDING, + validateInResponseTo: process.env.OVERLEAF_SAML_VALIDATE_IN_RESPONSE_TO, + requestIdExpirationPeriodMs: numFromEnv(process.env.OVERLEAF_SAML_REQUEST_ID_EXPIRATION_PERIOD_MS), + // cacheProvider: process.env.OVERLEAF_SAML_CACHE_PROVIDER, + logoutUrl: process.env.OVERLEAF_SAML_LOGOUT_URL, + logoutCallbackUrl: `${Settings.siteUrl.replace(/\/+$/, '')}/saml/logout/callback`, + additionalLogoutParams: JSON.parse(process.env.OVERLEAF_SAML_ADDITIONAL_LOGOUT_PARAMS || '{}'), + passReqToCallback: true, + } + try { + passport.use( + new SAMLStrategy( + samlOptions, + SAMLAuthenticationController.doPassportLogin, + SAMLAuthenticationController.doPassportLogout + ) + ) + callback(null) + } catch (error) { + callback(error) + } + }, + initPolicy() { + try { + PermissionsManager.registerCapability('change-password', { default : true }) + } catch (error) { + logger.info({}, error.message) + } + const samlPolicyValidator = async ({ user, subscription }) => { +// If user is not logged in, user.externalAuth is undefined, +// in this case allow to change password if the user has a hashedPassword + return user.externalAuth === 'saml' || (user.externalAuth === undefined && !user.hashedPassword) + } + try { + PermissionsManager.registerPolicy( + 'samlPolicy', + { 'change-password' : false }, + { validator: samlPolicyValidator } + ) + } catch (error) { + logger.info({}, error.message) + } + }, + async getGroupPolicyForUser(user, callback) { + try { + const userValidationMap = await PermissionsManager.promises.getUserValidationStatus({ + user, + groupPolicy : { 'samlPolicy' : true }, + subscription : null + }) + let groupPolicy = Object.fromEntries(userValidationMap) + callback(null, {'groupPolicy' : groupPolicy }) + } catch (error) { + callback(error) + } + }, +} + +export default SAMLModuleManager diff --git a/services/web/modules/authentication/saml/app/src/SAMLNonCsrfRouter.mjs b/services/web/modules/authentication/saml/app/src/SAMLNonCsrfRouter.mjs new file mode 100644 index 00000000000..c0d617b299b --- /dev/null +++ b/services/web/modules/authentication/saml/app/src/SAMLNonCsrfRouter.mjs @@ -0,0 +1,11 @@ +import logger from '@overleaf/logger' +import SAMLAuthenticationController from './SAMLAuthenticationController.mjs' + +export default { + apply(webRouter) { + logger.debug({}, 'Init SAML NonCsrfRouter') + webRouter.post('/saml/login/callback', SAMLAuthenticationController.passportLoginCallback) + webRouter.get ('/saml/logout/callback', SAMLAuthenticationController.passportLogoutCallback) + webRouter.post('/saml/logout/callback', SAMLAuthenticationController.passportLogoutCallback) + }, +} diff --git a/services/web/modules/authentication/saml/app/src/SAMLRouter.mjs b/services/web/modules/authentication/saml/app/src/SAMLRouter.mjs new file mode 100644 index 00000000000..3cd6e56e2da --- /dev/null +++ b/services/web/modules/authentication/saml/app/src/SAMLRouter.mjs @@ -0,0 +1,16 @@ +import logger from '@overleaf/logger' +import AuthenticationController from '../../../../../app/src/Features/Authentication/AuthenticationController.js' +import UserController from '../../../../../app/src/Features/User/UserController.js' +import SAMLAuthenticationController from './SAMLAuthenticationController.mjs' +import logout from '../../../logout.mjs' + +export default { + apply(webRouter) { + logger.debug({}, 'Init SAML router') + webRouter.get('/saml/login', SAMLAuthenticationController.passportLogin) + AuthenticationController.addEndpointToLoginWhitelist('/saml/login') + webRouter.get('/saml/meta', SAMLAuthenticationController.getSPMetadata) + AuthenticationController.addEndpointToLoginWhitelist('/saml/meta') + webRouter.post('/logout', logout, UserController.logout) + }, +} diff --git a/services/web/modules/authentication/saml/index.mjs b/services/web/modules/authentication/saml/index.mjs new file mode 100644 index 00000000000..2d6ee5706c6 --- /dev/null +++ b/services/web/modules/authentication/saml/index.mjs @@ -0,0 +1,18 @@ +let samlModule = {} +if (process.env.EXTERNAL_AUTH.includes('saml')) { + const { default: SAMLModuleManager } = await import('./app/src/SAMLModuleManager.mjs') + const { default: router } = await import('./app/src/SAMLRouter.mjs') + const { default: nonCsrfRouter } = await import('./app/src/SAMLNonCsrfRouter.mjs') + SAMLModuleManager.initSettings() + SAMLModuleManager.initPolicy() + samlModule = { + name: 'saml-authentication', + hooks: { + passportSetup: SAMLModuleManager.passportSetup, + getGroupPolicyForUser: SAMLModuleManager.getGroupPolicyForUser, + }, + router: router, + nonCsrfRouter: nonCsrfRouter, + } +} +export default samlModule diff --git a/services/web/modules/authentication/utils.mjs b/services/web/modules/authentication/utils.mjs new file mode 100644 index 00000000000..468dae32b15 --- /dev/null +++ b/services/web/modules/authentication/utils.mjs @@ -0,0 +1,42 @@ +import fs from 'fs' +function readFilesContentFromEnv(envVar) { +// envVar is either a file name: 'file.pem', or string with array: '["file.pem", "file2.pem"]' + if (!envVar) return undefined + try { + const parsedFileNames = JSON.parse(envVar) + return parsedFileNames.map(filename => fs.readFileSync(filename, 'utf8')) + } catch (error) { + if (error instanceof SyntaxError) { // failed to parse, envVar must be a file name + return fs.readFileSync(envVar, 'utf8') + } else { + throw error + } + } +} +function numFromEnv(env) { + return env ? Number(env) : undefined +} +function boolFromEnv(env) { + if (env === undefined || env === null) return undefined + if (typeof env === "string") { + const envLower = env.toLowerCase() + if (envLower === 'true') return true + if (envLower === 'false') return false + } + throw new Error("Invalid value for boolean envirionment variable") +} + +function splitFullName(fullName) { + fullName = fullName.trim(); + let lastSpaceIndex = fullName.lastIndexOf(' '); + let firstNames = fullName.substring(0, lastSpaceIndex).trim(); + let lastName = fullName.substring(lastSpaceIndex + 1).trim(); + return [firstNames, lastName]; +} + +export { + readFilesContentFromEnv, + numFromEnv, + boolFromEnv, + splitFullName, +} diff --git a/services/web/modules/ldap-authentication/app/src/AuthenticationControllerLdap.mjs b/services/web/modules/ldap-authentication/app/src/AuthenticationControllerLdap.mjs deleted file mode 100644 index 64fa4f5a96f..00000000000 --- a/services/web/modules/ldap-authentication/app/src/AuthenticationControllerLdap.mjs +++ /dev/null @@ -1,64 +0,0 @@ -import logger from '@overleaf/logger' -import LoginRateLimiter from '../../../../app/src/Features/Security/LoginRateLimiter.js' -import { handleAuthenticateErrors } from '../../../../app/src/Features/Authentication/AuthenticationErrors.js' -import AuthenticationController from '../../../../app/src/Features/Authentication/AuthenticationController.js' -import AuthenticationManagerLdap from './AuthenticationManagerLdap.mjs' - -const AuthenticationControllerLdap = { - async doPassportLdapLogin(req, ldapUser, done) { - let user, info - try { - ;({ user, info } = await AuthenticationControllerLdap._doPassportLdapLogin( - req, - ldapUser - )) - } catch (error) { - return done(error) - } - return done(undefined, user, info) - }, - async _doPassportLdapLogin(req, ldapUser) { - const { fromKnownDevice } = AuthenticationController.getAuditInfo(req) - const auditLog = { - ipAddress: req.ip, - info: { method: 'LDAP password login', fromKnownDevice }, - } - - let user, isPasswordReused - try { - user = await AuthenticationManagerLdap.promises.findOrCreateLdapUser(ldapUser, auditLog) - } catch (error) { - return { - user: false, - info: handleAuthenticateErrors(error, req), - } - } - if (user && AuthenticationController.captchaRequiredForLogin(req, user)) { - return { - user: false, - info: { - text: req.i18n.translate('cannot_verify_user_not_robot'), - type: 'error', - errorReason: 'cannot_verify_user_not_robot', - status: 400, - }, - } - } else if (user) { - // async actions - return { user, info: undefined } - } else { //something wrong - logger.debug({ email : ldapUser.mail }, 'failed LDAP log in') - return { - user: false, - info: { - type: 'error', - status: 500, - }, - } - } - }, -} - -export const { - doPassportLdapLogin, -} = AuthenticationControllerLdap diff --git a/services/web/modules/ldap-authentication/app/src/InitLdapSettings.mjs b/services/web/modules/ldap-authentication/app/src/InitLdapSettings.mjs deleted file mode 100644 index e7f312fc113..00000000000 --- a/services/web/modules/ldap-authentication/app/src/InitLdapSettings.mjs +++ /dev/null @@ -1,17 +0,0 @@ -import Settings from '@overleaf/settings' - -function initLdapSettings() { - Settings.ldap = { - enable: true, - placeholder: process.env.OVERLEAF_LDAP_PLACEHOLDER || 'Username', - attEmail: process.env.OVERLEAF_LDAP_EMAIL_ATT || 'mail', - attFirstName: process.env.OVERLEAF_LDAP_FIRST_NAME_ATT, - attLastName: process.env.OVERLEAF_LDAP_LAST_NAME_ATT, - attName: process.env.OVERLEAF_LDAP_NAME_ATT, - attAdmin: process.env.OVERLEAF_LDAP_IS_ADMIN_ATT, - valAdmin: process.env.OVERLEAF_LDAP_IS_ADMIN_ATT_VALUE, - updateUserDetailsOnLogin: String(process.env.OVERLEAF_LDAP_UPDATE_USER_DETAILS_ON_LOGIN ).toLowerCase() === 'true', - } -} - -export default initLdapSettings diff --git a/services/web/modules/ldap-authentication/app/src/LdapContacts.mjs b/services/web/modules/ldap-authentication/app/src/LdapContacts.mjs deleted file mode 100644 index c4093b8684e..00000000000 --- a/services/web/modules/ldap-authentication/app/src/LdapContacts.mjs +++ /dev/null @@ -1,136 +0,0 @@ -import Settings from '@overleaf/settings' -import logger from '@overleaf/logger' -import passport from 'passport' -import ldapjs from 'ldapauth-fork/node_modules/ldapjs/lib/index.js' -import UserGetter from '../../../../app/src/Features/User/UserGetter.js' -import { splitFullName } from './AuthenticationManagerLdap.mjs' - -async function fetchLdapContacts(userId, contacts) { - if (!Settings.ldap?.enable || !process.env.OVERLEAF_LDAP_CONTACTS_FILTER) { - return [] - } - - const ldapOpts = passport._strategy('custom-fail-ldapauth').options.server - const { attEmail, attFirstName = "", attLastName = "", attName = "" } = Settings.ldap - const { - url, - timeout, - connectTimeout, - tlsOptions, - starttls, - bindDN, - bindCredentials, - } = ldapOpts - const searchBase = process.env.OVERLEAF_LDAP_CONTACTS_SEARCH_BASE || ldapOpts.searchBase - const searchScope = process.env.OVERLEAF_LDAP_CONTACTS_SEARCH_SCOPE || 'sub' - const ldapConfig = { url, timeout, connectTimeout, tlsOptions } - - let ldapUsers - const client = ldapjs.createClient(ldapConfig) - try { - if (starttls) { - await _upgradeToTLS(client, tlsOptions) - } - await _bindLdap(client, bindDN, bindCredentials) - - const filter = await _formContactsSearchFilter(client, ldapOpts, userId, process.env.OVERLEAF_LDAP_CONTACTS_FILTER) - const searchOptions = { scope: searchScope, attributes: [attEmail, attFirstName, attLastName, attName], filter } - - ldapUsers = await _searchLdap(client, searchBase, searchOptions) - } catch (err) { - logger.warn({ err }, 'error in fetchLdapContacts') - return [] - } finally { - client.unbind() - } - - const newLdapContacts = ldapUsers.reduce((acc, ldapUser) => { - const email = Array.isArray(ldapUser[attEmail]) - ? ldapUser[attEmail][0]?.toLowerCase() - : ldapUser[attEmail]?.toLowerCase() - if (!email) return acc - if (!contacts.some(contact => contact.email === email)) { - let nameParts = ["",""] - if ((!attFirstName || !attLastName) && attName) { - nameParts = splitFullName(ldapUser[attName] || "") - } - const firstName = attFirstName ? (ldapUser[attFirstName] || "") : nameParts[0] - const lastName = attLastName ? (ldapUser[attLastName] || "") : nameParts[1] - acc.push({ - first_name: firstName, - last_name: lastName, - email: email, - type: 'user', - }) - } - return acc - }, []) - - return newLdapContacts.sort((a, b) => - a.last_name.localeCompare(b.last_name) || - a.first_name.localeCompare(a.first_name) || - a.email.localeCompare(b.email) - ) -} - -function _upgradeToTLS(client, tlsOptions) { - return new Promise((resolve, reject) => { - client.on('error', error => reject(new Error(`LDAP client error: ${error}`))) - client.on('connect', () => { - client.starttls(tlsOptions, null, error => { - if (error) { - reject(new Error(`StartTLS error: ${error}`)) - } else { - resolve() - } - }) - }) - }) -} - -function _bindLdap(client, bindDN, bindCredentials) { - return new Promise((resolve, reject) => { - client.bind(bindDN, bindCredentials, error => { - if (error) { - reject(error) - } else { - resolve() - } - }) - }) -} - -function _searchLdap(client, baseDN, options) { - return new Promise((resolve, reject) => { - const searchEntries = [] - client.search(baseDN, options, (error, res) => { - if (error) { - reject(error) - } else { - res.on('searchEntry', entry => searchEntries.push(entry.object)) - res.on('error', reject) - res.on('end', () => resolve(searchEntries)) - } - }) - }) -} - -async function _formContactsSearchFilter(client, ldapOpts, userId, contactsFilter) { - const searchProperty = process.env.OVERLEAF_LDAP_CONTACTS_PROPERTY - if (!searchProperty) { - return contactsFilter - } - const email = await UserGetter.promises.getUserEmail(userId) - const searchOptions = { - scope: ldapOpts.searchScope, - attributes: [searchProperty], - filter: `(${Settings.ldap.attEmail}=${email})`, - } - const searchBase = ldapOpts.searchBase - const ldapUser = (await _searchLdap(client, searchBase, searchOptions))[0] - const searchPropertyValue = ldapUser ? ldapUser[searchProperty] - : process.env.OVERLEAF_LDAP_CONTACTS_NON_LDAP_VALUE || 'IMATCHNOTHING' - return contactsFilter.replace(/{{userProperty}}/g, searchPropertyValue) -} - -export default fetchLdapContacts diff --git a/services/web/modules/ldap-authentication/app/src/LdapStrategy.mjs b/services/web/modules/ldap-authentication/app/src/LdapStrategy.mjs deleted file mode 100644 index b07dc3f3bdd..00000000000 --- a/services/web/modules/ldap-authentication/app/src/LdapStrategy.mjs +++ /dev/null @@ -1,78 +0,0 @@ -import fs from 'fs' -import passport from 'passport' -import Settings from '@overleaf/settings' -import { doPassportLdapLogin } from './AuthenticationControllerLdap.mjs' -import { Strategy as LdapStrategy } from 'passport-ldapauth' - -function _readFilesContentFromEnv(envVar) { -// envVar is either a file name: 'file.pem', or string with array: '["file.pem", "file2.pem"]' - if (!envVar) return undefined - try { - const parsedFileNames = JSON.parse(envVar) - return parsedFileNames.map(filename => fs.readFileSync(filename, 'utf8')) - } catch (error) { - if (error instanceof SyntaxError) { // failed to parse, envVar must be a file name - return fs.readFileSync(envVar, 'utf8') - } else { - throw error - } - } -} - -// custom responses on authentication failure -class CustomFailLdapStrategy extends LdapStrategy { - constructor(options, validate) { - super(options, validate); - this.name = 'custom-fail-ldapauth' - } - authenticate(req, options) { - const defaultFail = this.fail.bind(this) - this.fail = function(info, status) { - info.type = 'error' - info.key = 'invalid-password-retry-or-reset' - info.status = 401 - return defaultFail(info, status) - }.bind(this) - super.authenticate(req, options) - } -} - -const ldapServerOpts = { - url: process.env.OVERLEAF_LDAP_URL, - bindDN: process.env.OVERLEAF_LDAP_BIND_DN || "", - bindCredentials: process.env.OVERLEAF_LDAP_BIND_CREDENTIALS || "", - bindProperty: process.env.OVERLEAF_LDAP_BIND_PROPERTY, - searchBase: process.env.OVERLEAF_LDAP_SEARCH_BASE, - searchFilter: process.env.OVERLEAF_LDAP_SEARCH_FILTER, - searchScope: process.env.OVERLEAF_LDAP_SEARCH_SCOPE || 'sub', - searchAttributes: JSON.parse(process.env.OVERLEAF_LDAP_SEARCH_ATTRIBUTES || '[]'), - groupSearchBase: process.env.OVERLEAF_LDAP_ADMIN_SEARCH_BASE, - groupSearchFilter: process.env.OVERLEAF_LDAP_ADMIN_SEARCH_FILTER, - groupSearchScope: process.env.OVERLEAF_LDAP_ADMIN_SEARCH_SCOPE || 'sub', - groupSearchAttributes: ["dn"], - groupDnProperty: process.env.OVERLEAF_LDAP_ADMIN_DN_PROPERTY, - cache: String(process.env.OVERLEAF_LDAP_CACHE).toLowerCase() === 'true', - timeout: process.env.OVERLEAF_LDAP_TIMEOUT ? Number(process.env.OVERLEAF_LDAP_TIMEOUT) : undefined, - connectTimeout: process.env.OVERLEAF_LDAP_CONNECT_TIMEOUT ? Number(process.env.OVERLEAF_LDAP_CONNECT_TIMEOUT) : undefined, - starttls: String(process.env.OVERLEAF_LDAP_STARTTLS).toLowerCase() === 'true', - tlsOptions: { - ca: _readFilesContentFromEnv(process.env.OVERLEAF_LDAP_TLS_OPTS_CA_PATH), - rejectUnauthorized: String(process.env.OVERLEAF_LDAP_TLS_OPTS_REJECT_UNAUTH).toLowerCase() === 'true', - } -} - -function addLdapStrategy(passport) { - passport.use( - new CustomFailLdapStrategy( - { - server: ldapServerOpts, - passReqToCallback: true, - usernameField: 'email', - passwordField: 'password', - }, - doPassportLdapLogin - ) - ) -} - -export default addLdapStrategy diff --git a/services/web/modules/ldap-authentication/index.mjs b/services/web/modules/ldap-authentication/index.mjs deleted file mode 100644 index f56d7ffee08..00000000000 --- a/services/web/modules/ldap-authentication/index.mjs +++ /dev/null @@ -1,30 +0,0 @@ -import initLdapSettings from './app/src/InitLdapSettings.mjs' -import addLdapStrategy from './app/src/LdapStrategy.mjs' -import fetchLdapContacts from './app/src/LdapContacts.mjs' - -let ldapModule = {}; -if (process.env.EXTERNAL_AUTH === 'ldap') { - initLdapSettings() - ldapModule = { - name: 'ldap-authentication', - hooks: { - passportSetup: function (passport, callback) { - try { - addLdapStrategy(passport) - callback(null) - } catch (error) { - callback(error) - } - }, - getContacts: async function (userId, contacts, callback) { - try { - const newLdapContacts = await fetchLdapContacts(userId, contacts) - callback(null, newLdapContacts) - } catch (error) { - callback(error) - } - }, - } - } -} -export default ldapModule diff --git a/services/web/modules/saml-authentication/app/src/AuthenticationManagerSaml.mjs b/services/web/modules/saml-authentication/app/src/AuthenticationManagerSaml.mjs deleted file mode 100644 index 47d97f3019c..00000000000 --- a/services/web/modules/saml-authentication/app/src/AuthenticationManagerSaml.mjs +++ /dev/null @@ -1,60 +0,0 @@ -import Settings from '@overleaf/settings' -import UserCreator from '../../../../app/src/Features/User/UserCreator.js' -import { User } from '../../../../app/src/models/User.js' - -const AuthenticationManagerSaml = { - async findOrCreateSamlUser(profile, auditLog) { - const { - attEmail, - attFirstName, - attLastName, - attAdmin, - valAdmin, - updateUserDetailsOnLogin, - } = Settings.saml - const email = Array.isArray(profile[attEmail]) - ? profile[attEmail][0].toLowerCase() - : profile[attEmail].toLowerCase() - const firstName = attFirstName ? profile[attFirstName] : "" - const lastName = attLastName ? profile[attLastName] : email - let isAdmin = false - if( attAdmin && valAdmin ) { - isAdmin = (Array.isArray(profile[attAdmin]) ? profile[attAdmin].includes(valAdmin) : - profile[attAdmin] === valAdmin) - } - let user = await User.findOne({ 'email': email }).exec() - if( !user ) { - user = await UserCreator.promises.createNewUser( - { - email: email, - first_name: firstName, - last_name: lastName, - isAdmin: isAdmin, - holdingAccount: false, - } - ) - await User.updateOne( - { _id: user._id }, - { $set : { 'emails.0.confirmedAt' : Date.now() } } - ).exec() //email of saml user is confirmed - } - let userDetails = updateUserDetailsOnLogin ? { first_name : firstName, last_name: lastName } : {} - if( attAdmin && valAdmin ) { - user.isAdmin = isAdmin - userDetails.isAdmin = isAdmin - } - const result = await User.updateOne( - { _id: user._id, loginEpoch: user.loginEpoch }, { $inc: { loginEpoch: 1 }, $set: userDetails }, - {} - ).exec() - - if (result.modifiedCount !== 1) { - throw new ParallelLoginError() - } - return user - }, -} - -export default { - promises: AuthenticationManagerSaml, -} diff --git a/services/web/modules/saml-authentication/app/src/InitSamlSettings.mjs b/services/web/modules/saml-authentication/app/src/InitSamlSettings.mjs deleted file mode 100644 index 441f9033af9..00000000000 --- a/services/web/modules/saml-authentication/app/src/InitSamlSettings.mjs +++ /dev/null @@ -1,16 +0,0 @@ -import Settings from '@overleaf/settings' - -function initSamlSettings() { - Settings.saml = { - enable: true, - identityServiceName: process.env.OVERLEAF_SAML_IDENTITY_SERVICE_NAME || 'Login with SAML IdP', - attEmail: process.env.OVERLEAF_SAML_EMAIL_FIELD || 'nameID', - attFirstName: process.env.OVERLEAF_SAML_FIRST_NAME_FIELD || 'givenName', - attLastName: process.env.OVERLEAF_SAML_LAST_NAME_FIELD || 'lastName', - attAdmin: process.env.OVERLEAF_SAML_IS_ADMIN_FIELD, - valAdmin: process.env.OVERLEAF_SAML_IS_ADMIN_FIELD_VALUE, - updateUserDetailsOnLogin: String(process.env.OVERLEAF_SAML_UPDATE_USER_DETAILS_ON_LOGIN).toLowerCase() === 'true', - } -} - -export default initSamlSettings diff --git a/services/web/modules/saml-authentication/app/src/SamlNonCsrfRouter.mjs b/services/web/modules/saml-authentication/app/src/SamlNonCsrfRouter.mjs deleted file mode 100644 index 65b42c92aef..00000000000 --- a/services/web/modules/saml-authentication/app/src/SamlNonCsrfRouter.mjs +++ /dev/null @@ -1,12 +0,0 @@ -import logger from '@overleaf/logger' -import { passportSamlLogin, passportSamlIdPLogout } from './AuthenticationControllerSaml.mjs' - -export default { - apply(webRouter) { - logger.debug({}, 'Init SAML NonCsrfRouter') - webRouter.get('/saml/login/callback', passportSamlLogin) - webRouter.post('/saml/login/callback', passportSamlLogin) - webRouter.get('/saml/logout/callback', passportSamlIdPLogout) - webRouter.post('/saml/logout/callback', passportSamlIdPLogout) - }, -} diff --git a/services/web/modules/saml-authentication/app/src/SamlRouter.mjs b/services/web/modules/saml-authentication/app/src/SamlRouter.mjs deleted file mode 100644 index 9ee36779011..00000000000 --- a/services/web/modules/saml-authentication/app/src/SamlRouter.mjs +++ /dev/null @@ -1,14 +0,0 @@ -import logger from '@overleaf/logger' -import AuthenticationController from '../../../../app/src/Features/Authentication/AuthenticationController.js' -import { passportSamlAuthWithIdP, passportSamlSPLogout, passportSamlMetadata} from './AuthenticationControllerSaml.mjs' - -export default { - apply(webRouter) { - logger.debug({}, 'Init SAML router') - webRouter.get('/saml/login', passportSamlAuthWithIdP) - AuthenticationController.addEndpointToLoginWhitelist('/saml/login') - webRouter.post('/saml/logout', AuthenticationController.requireLogin(), passportSamlSPLogout) - webRouter.get('/saml/meta', passportSamlMetadata) - AuthenticationController.addEndpointToLoginWhitelist('/saml/meta') - }, -} diff --git a/services/web/modules/saml-authentication/app/src/SamlStrategy.mjs b/services/web/modules/saml-authentication/app/src/SamlStrategy.mjs deleted file mode 100644 index 3a16459f981..00000000000 --- a/services/web/modules/saml-authentication/app/src/SamlStrategy.mjs +++ /dev/null @@ -1,62 +0,0 @@ -import fs from 'fs' -import passport from 'passport' -import Settings from '@overleaf/settings' -import { doPassportSamlLogin, doPassportSamlLogout } from './AuthenticationControllerSaml.mjs' -import { Strategy as SamlStrategy } from '@node-saml/passport-saml' - -function _readFilesContentFromEnv(envVar) { -// envVar is either a file name: 'file.pem', or string with array: '["file.pem", "file2.pem"]' - if (!envVar) return undefined - try { - const parsedFileNames = JSON.parse(envVar) - return parsedFileNames.map(filename => fs.readFileSync(filename, 'utf8')) - } catch (error) { - if (error instanceof SyntaxError) { // failed to parse, envVar must be a file name - return fs.readFileSync(envVar, 'utf8') - } else { - throw error - } - } -} - -const samlOptions = { - entryPoint: process.env.OVERLEAF_SAML_ENTRYPOINT, - callbackUrl: process.env.OVERLEAF_SAML_CALLBACK_URL, - issuer: process.env.OVERLEAF_SAML_ISSUER, - audience: process.env.OVERLEAF_SAML_AUDIENCE, - cert: _readFilesContentFromEnv(process.env.OVERLEAF_SAML_IDP_CERT), - signingCert: _readFilesContentFromEnv(process.env.OVERLEAF_SAML_PUBLIC_CERT), - privateKey: _readFilesContentFromEnv(process.env.OVERLEAF_SAML_PRIVATE_KEY), - decryptionCert: _readFilesContentFromEnv(process.env.OVERLEAF_SAML_DECRYPTION_CERT), - decryptionPvk: _readFilesContentFromEnv(process.env.OVERLEAF_SAML_DECRYPTION_PVK), - signatureAlgorithm: process.env.OVERLEAF_SAML_SIGNATURE_ALGORITHM, - additionalParams: JSON.parse(process.env.OVERLEAF_SAML_ADDITIONAL_PARAMS || '{}'), - additionalAuthorizeParams: JSON.parse(process.env.OVERLEAF_SAML_ADDITIONAL_AUTHORIZE_PARAMS || '{}'), - identifierFormat: process.env.OVERLEAF_SAML_IDENTIFIER_FORMAT, - acceptedClockSkewMs: process.env.OVERLEAF_SAML_ACCEPTED_CLOCK_SKEW_MS ? Number(process.env.OVERLEAF_SAML_ACCEPTED_CLOCK_SKEW_MS) : undefined, - attributeConsumingServiceIndex: process.env.OVERLEAF_SAML_ATTRIBUTE_CONSUMING_SERVICE_INDEX, - authnContext: process.env.OVERLEAF_SAML_AUTHN_CONTEXT ? JSON.parse(process.env.OVERLEAF_SAML_AUTHN_CONTEXT) : undefined, - forceAuthn: String(process.env.OVERLEAF_SAML_FORCE_AUTHN).toLowerCase() === 'true', - disableRequestedAuthnContext: String(process.env.OVERLEAF_SAML_DISABLE_REQUESTED_AUTHN_CONTEXT).toLowerCase() === 'true', - skipRequestCompression: process.env.OVERLEAF_SAML_AUTHN_REQUEST_BINDING === 'HTTP-POST', // compression should be skipped iff authnRequestBinding is POST - authnRequestBinding: process.env.OVERLEAF_SAML_AUTHN_REQUEST_BINDING, - validateInResponseTo: process.env.OVERLEAF_SAML_VALIDATE_IN_RESPONSE_TO, - requestIdExpirationPeriodMs: process.env.OVERLEAF_SAML_REQUEST_ID_EXPIRATION_PERIOD_MS ? Number(process.env.OVERLEAF_SAML_REQUEST_ID_EXPIRATION_PERIOD_MS) : undefined, -// cacheProvider: process.env.OVERLEAF_SAML_CACHE_PROVIDER, - logoutUrl: process.env.OVERLEAF_SAML_LOGOUT_URL, - logoutCallbackUrl: process.env.OVERLEAF_SAML_LOGOUT_CALLBACK_URL, - additionalLogoutParams: JSON.parse(process.env.OVERLEAF_SAML_ADDITIONAL_LOGOUT_PARAMS || '{}'), - passReqToCallback: true, -} - -function addSamlStrategy(passport) { - passport.use( - new SamlStrategy( - samlOptions, - doPassportSamlLogin, - doPassportSamlLogout - ) - ) -} - -export default addSamlStrategy diff --git a/services/web/modules/saml-authentication/index.mjs b/services/web/modules/saml-authentication/index.mjs deleted file mode 100644 index 35ea70283fe..00000000000 --- a/services/web/modules/saml-authentication/index.mjs +++ /dev/null @@ -1,26 +0,0 @@ -import initSamlSettings from './app/src/InitSamlSettings.mjs' -import addSamlStrategy from './app/src/SamlStrategy.mjs' -import SamlRouter from './app/src/SamlRouter.mjs' -import SamlNonCsrfRouter from './app/src/SamlNonCsrfRouter.mjs' - -let samlModule = {}; - -if (process.env.EXTERNAL_AUTH === 'saml') { - initSamlSettings() - samlModule = { - name: 'saml-authentication', - hooks: { - passportSetup: function (passport, callback) { - try { - addSamlStrategy(passport) - callback(null) - } catch (error) { - callback(error) - } - }, - }, - router: SamlRouter, - nonCsrfRouter: SamlNonCsrfRouter, - } -} -export default samlModule diff --git a/services/web/package.json b/services/web/package.json index d9bb858e523..0891dbc3eca 100644 --- a/services/web/package.json +++ b/services/web/package.json @@ -170,6 +170,7 @@ "passport-ldapauth": "^2.1.4", "passport-local": "^1.0.0", "passport-oauth2": "^1.5.0", + "passport-openidconnect": "^0.1.2", "passport-orcid": "0.0.4", "pug": "^3.0.3", "pug-runtime": "^3.0.1", From cfab2e7fb9352cef8fa67e2f2cddf9087c4f5691 Mon Sep 17 00:00:00 2001 From: yu-i-i Date: Mon, 27 Jan 2025 04:58:23 +0100 Subject: [PATCH 010/106] Re-export `doLogout` (was removed from exports in commit b9fb636). --- services/web/app/src/Features/User/UserController.mjs | 1 + .../oidc/app/src/OIDCAuthenticationController.mjs | 2 +- .../saml/app/src/SAMLAuthenticationController.mjs | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/services/web/app/src/Features/User/UserController.mjs b/services/web/app/src/Features/User/UserController.mjs index 37cc016f9c7..61593e7b11e 100644 --- a/services/web/app/src/Features/User/UserController.mjs +++ b/services/web/app/src/Features/User/UserController.mjs @@ -546,4 +546,5 @@ export default { expireDeletedUsersAfterDuration: expressify(expireDeletedUsersAfterDuration), ensureAffiliationMiddleware: expressify(ensureAffiliationMiddleware), ensureAffiliation, + doLogout, } diff --git a/services/web/modules/authentication/oidc/app/src/OIDCAuthenticationController.mjs b/services/web/modules/authentication/oidc/app/src/OIDCAuthenticationController.mjs index 42c01e712f9..0b8dc501e0e 100644 --- a/services/web/modules/authentication/oidc/app/src/OIDCAuthenticationController.mjs +++ b/services/web/modules/authentication/oidc/app/src/OIDCAuthenticationController.mjs @@ -158,7 +158,7 @@ const OIDCAuthenticationController = { async passportLogout(req, res, next) { // TODO: instead of storing idToken in session, use refreshToken to obtain a new idToken? const idTokenHint = req.session.idToken - await UserController.promises.doLogout(req) + await UserController.doLogout(req) const logoutUrl = process.env.OVERLEAF_OIDC_LOGOUT_URL const redirectUri = Settings.siteUrl res.redirect(`${logoutUrl}?id_token_hint=${idTokenHint}&post_logout_redirect_uri=${encodeURIComponent(redirectUri)}`) diff --git a/services/web/modules/authentication/saml/app/src/SAMLAuthenticationController.mjs b/services/web/modules/authentication/saml/app/src/SAMLAuthenticationController.mjs index ac0e5398b28..3ed834608ff 100644 --- a/services/web/modules/authentication/saml/app/src/SAMLAuthenticationController.mjs +++ b/services/web/modules/authentication/saml/app/src/SAMLAuthenticationController.mjs @@ -102,7 +102,7 @@ const SAMLAuthenticationController = { }, async passportLogout(req, res, next) { passport._strategy('saml').logout(req, async (err, url) => { - await UserController.promises.doLogout(req) + await UserController.doLogout(req) if (err) return next(err) res.redirect(url) }) From eec6b59cb5e620cdbcb96f1f2b73cc77b9f985f9 Mon Sep 17 00:00:00 2001 From: yu-i-i Date: Thu, 6 Feb 2025 12:12:03 +0100 Subject: [PATCH 011/106] Add ENV variables to control SAML signature validation --- .../modules/authentication/saml/app/src/SAMLModuleManager.mjs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/services/web/modules/authentication/saml/app/src/SAMLModuleManager.mjs b/services/web/modules/authentication/saml/app/src/SAMLModuleManager.mjs index c7efdef214c..29e9ae52cdd 100644 --- a/services/web/modules/authentication/saml/app/src/SAMLModuleManager.mjs +++ b/services/web/modules/authentication/saml/app/src/SAMLModuleManager.mjs @@ -46,6 +46,8 @@ const SAMLModuleManager = { logoutUrl: process.env.OVERLEAF_SAML_LOGOUT_URL, logoutCallbackUrl: `${Settings.siteUrl.replace(/\/+$/, '')}/saml/logout/callback`, additionalLogoutParams: JSON.parse(process.env.OVERLEAF_SAML_ADDITIONAL_LOGOUT_PARAMS || '{}'), + wantAssertionsSigned: boolFromEnv(process.env.OVERLEAF_SAML_WANT_ASSERTIONS_SIGNED), + wantAuthnResponseSigned: boolFromEnv(process.env.OVERLEAF_SAML_WANT_AUTHN_RESPONSE_SIGNED), passReqToCallback: true, } try { From 7403225fea857251c83a18691f50eec495bc710d Mon Sep 17 00:00:00 2001 From: yu-i-i Date: Sat, 22 Feb 2025 03:26:25 +0100 Subject: [PATCH 012/106] Whitelist /oidc/login endpoint, fixes #21 --- .../web/modules/authentication/oidc/app/src/OIDCRouter.mjs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/services/web/modules/authentication/oidc/app/src/OIDCRouter.mjs b/services/web/modules/authentication/oidc/app/src/OIDCRouter.mjs index 519fa5043a8..0857e41889c 100644 --- a/services/web/modules/authentication/oidc/app/src/OIDCRouter.mjs +++ b/services/web/modules/authentication/oidc/app/src/OIDCRouter.mjs @@ -1,5 +1,6 @@ import logger from '@overleaf/logger' import UserController from '../../../../../app/src/Features/User/UserController.js' +import AuthenticationController from '../../../../../app/src/Features/Authentication/AuthenticationController.js' import OIDCAuthenticationController from './OIDCAuthenticationController.mjs' import logout from '../../../logout.mjs' @@ -7,7 +8,9 @@ export default { apply(webRouter) { logger.debug({}, 'Init OIDC router') webRouter.get('/oidc/login', OIDCAuthenticationController.passportLogin) + AuthenticationController.addEndpointToLoginWhitelist('/oidc/login') webRouter.get('/oidc/login/callback', OIDCAuthenticationController.passportLoginCallback) + AuthenticationController.addEndpointToLoginWhitelist('/oidc/login/callback') webRouter.get('/oidc/logout/callback', OIDCAuthenticationController.passportLogoutCallback) webRouter.post('/user/oauth-unlink', OIDCAuthenticationController.unlinkAccount) webRouter.post('/logout', logout, UserController.logout) From 84820c2b81dd9cdee299941fdbf6c8e07919d4a6 Mon Sep 17 00:00:00 2001 From: yu-i-i Date: Sat, 8 Mar 2025 18:23:19 +0100 Subject: [PATCH 013/106] Fix glitches in symbol palette after switching to Bootstrap 5 --- .../components/symbol-palette-close-button.js | 8 ++--- .../components/symbol-palette-content.js | 5 +--- .../components/symbol-palette-info-link.js | 29 ------------------- .../components/symbol-palette-item.js | 2 +- .../stylesheets/modules/symbol-palette.scss | 4 +-- 5 files changed, 7 insertions(+), 41 deletions(-) delete mode 100644 services/web/frontend/js/features/symbol-palette/components/symbol-palette-info-link.js diff --git a/services/web/frontend/js/features/symbol-palette/components/symbol-palette-close-button.js b/services/web/frontend/js/features/symbol-palette/components/symbol-palette-close-button.js index c472c315868..6c776d1e248 100644 --- a/services/web/frontend/js/features/symbol-palette/components/symbol-palette-close-button.js +++ b/services/web/frontend/js/features/symbol-palette/components/symbol-palette-close-button.js @@ -5,14 +5,12 @@ export default function SymbolPaletteCloseButton() { const { toggleSymbolPalette } = useEditorContext() return ( +
+
) } - diff --git a/services/web/frontend/js/features/symbol-palette/components/symbol-palette-content.js b/services/web/frontend/js/features/symbol-palette/components/symbol-palette-content.js index 8537e145851..a1709877934 100644 --- a/services/web/frontend/js/features/symbol-palette/components/symbol-palette-content.js +++ b/services/web/frontend/js/features/symbol-palette/components/symbol-palette-content.js @@ -9,7 +9,6 @@ import { buildCategorisedSymbols, createCategories } from '../utils/categories' import SymbolPaletteSearch from './symbol-palette-search' import SymbolPaletteBody from './symbol-palette-body' import SymbolPaletteTabs from './symbol-palette-tabs' -// import SymbolPaletteInfoLink from './symbol-palette-info-link' import SymbolPaletteCloseButton from './symbol-palette-close-button' import '@reach/tabs/styles.css' @@ -68,9 +67,7 @@ export default function SymbolPaletteContent({ handleSelect }) {
-
- {/* Useless button (uncomment if you see any sense in it) */} - {/* */} +
diff --git a/services/web/frontend/js/features/symbol-palette/components/symbol-palette-info-link.js b/services/web/frontend/js/features/symbol-palette/components/symbol-palette-info-link.js deleted file mode 100644 index ba56cf2b106..00000000000 --- a/services/web/frontend/js/features/symbol-palette/components/symbol-palette-info-link.js +++ /dev/null @@ -1,29 +0,0 @@ -import { Button, OverlayTrigger, Tooltip } from 'react-bootstrap' -import { useTranslation } from 'react-i18next' - -export default function SymbolPaletteInfoLink() { - const { t } = useTranslation() - - return ( - - {t('find_out_more_about_latex_symbols')} - - } - > - - - ) -} diff --git a/services/web/frontend/js/features/symbol-palette/components/symbol-palette-item.js b/services/web/frontend/js/features/symbol-palette/components/symbol-palette-item.js index a892f33cf8b..e1fca8c434d 100644 --- a/services/web/frontend/js/features/symbol-palette/components/symbol-palette-item.js +++ b/services/web/frontend/js/features/symbol-palette/components/symbol-palette-item.js @@ -1,5 +1,5 @@ import { useEffect, useRef } from 'react' -import { OverlayTrigger, Tooltip } from 'react-bootstrap' +import { OverlayTrigger, Tooltip } from 'react-bootstrap-5' import PropTypes from 'prop-types' export default function SymbolPaletteItem({ diff --git a/services/web/frontend/stylesheets/modules/symbol-palette.scss b/services/web/frontend/stylesheets/modules/symbol-palette.scss index 66c9f7169e7..9a6aa7e0319 100644 --- a/services/web/frontend/stylesheets/modules/symbol-palette.scss +++ b/services/web/frontend/stylesheets/modules/symbol-palette.scss @@ -157,8 +157,8 @@ } .symbol-palette-close-button { - margin-top: var(--spacing-04); - margin-left: var(--spacing-05); + margin-top: var(--spacing-05); + margin-left: var(--spacing-02); margin-right: var(--spacing-03); .symbol-palette-unavailable & { From 26221c53b06a2558d6176e4ddd684e3634b25c26 Mon Sep 17 00:00:00 2001 From: yu-i-i Date: Mon, 10 Mar 2025 05:55:01 +0100 Subject: [PATCH 014/106] See upstream commit 42ee56e --- services/web/modules/authentication/ldap/app/src/LDAPRouter.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/web/modules/authentication/ldap/app/src/LDAPRouter.mjs b/services/web/modules/authentication/ldap/app/src/LDAPRouter.mjs index 44d9d373d2d..d2bbb352369 100644 --- a/services/web/modules/authentication/ldap/app/src/LDAPRouter.mjs +++ b/services/web/modules/authentication/ldap/app/src/LDAPRouter.mjs @@ -10,7 +10,7 @@ export default { logger.debug({}, 'Init LDAP router') webRouter.post('/login', RateLimiterMiddleware.rateLimit(overleafLoginRateLimiter), // rate limit IP (20 / 60s) - RateLimiterMiddleware.loginRateLimitEmail, // rate limit email (10 / 120s) + RateLimiterMiddleware.loginRateLimitEmail(), // rate limit email (10 / 120s) CaptchaMiddleware.validateCaptcha('login'), LDAPAuthenticationController.passportLogin, AuthenticationController.passportLogin, From 4b502a58aed5e2088b8651a7e58283fbcb26b6c9 Mon Sep 17 00:00:00 2001 From: yu-i-i Date: Mon, 10 Mar 2025 06:37:50 +0100 Subject: [PATCH 015/106] Make OVERLEAF_OIDC_USER_ID_FIELD support 'email' as a value --- .../authentication/oidc/app/src/OIDCAuthenticationManager.mjs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/web/modules/authentication/oidc/app/src/OIDCAuthenticationManager.mjs b/services/web/modules/authentication/oidc/app/src/OIDCAuthenticationManager.mjs index 56ec2e5455d..5295ce63d07 100644 --- a/services/web/modules/authentication/oidc/app/src/OIDCAuthenticationManager.mjs +++ b/services/web/modules/authentication/oidc/app/src/OIDCAuthenticationManager.mjs @@ -13,8 +13,8 @@ const OIDCAuthenticationManager = { updateUserDetailsOnLogin, providerId, } = Settings.oidc - const oidcUserId = profile[attUserId] const email = profile.emails[0].value + const oidcUserId = (attUserId === 'email') ? email : profile[attUserId] const firstName = profile.name?.givenName || "" const lastName = profile.name?.familyName || "" let isAdmin = false @@ -83,7 +83,7 @@ const OIDCAuthenticationManager = { attUserId, providerId, } = Settings.oidc - const oidcUserId = profile[attUserId] + const oidcUserId = (attUserId === 'email') ? profile.emails[0].value : profile[attUserId] const oidcUserData = null // Possibly it can be used later await ThirdPartyIdentityManager.promises.link(userId, providerId, oidcUserId, oidcUserData, auditLog) }, From 96137daf3dd45851f4a99fa201aa1adc159e6601 Mon Sep 17 00:00:00 2001 From: yu-i-i Date: Thu, 3 Apr 2025 23:26:54 +0200 Subject: [PATCH 016/106] Symbol palette: switch to 'OL' UI components and apply minor cosmetic changes --- .../components/symbol-palette-close-button.js | 13 ++++++---- .../components/symbol-palette-content.js | 5 ++-- .../components/symbol-palette-item.js | 25 +++++++++++-------- .../components/symbol-palette-search.js | 9 ++++--- .../stylesheets/modules/symbol-palette.scss | 2 ++ 5 files changed, 32 insertions(+), 22 deletions(-) diff --git a/services/web/frontend/js/features/symbol-palette/components/symbol-palette-close-button.js b/services/web/frontend/js/features/symbol-palette/components/symbol-palette-close-button.js index 6c776d1e248..839b5d1cd55 100644 --- a/services/web/frontend/js/features/symbol-palette/components/symbol-palette-close-button.js +++ b/services/web/frontend/js/features/symbol-palette/components/symbol-palette-close-button.js @@ -1,16 +1,19 @@ -import { Button } from 'react-bootstrap' import { useEditorContext } from '../../../shared/context/editor-context' +import { useTranslation } from 'react-i18next' export default function SymbolPaletteCloseButton() { const { toggleSymbolPalette } = useEditorContext() + const { t } = useTranslation() return ( -
- +
) } diff --git a/services/web/frontend/js/features/symbol-palette/components/symbol-palette-content.js b/services/web/frontend/js/features/symbol-palette/components/symbol-palette-content.js index a1709877934..cb5a9e3029c 100644 --- a/services/web/frontend/js/features/symbol-palette/components/symbol-palette-content.js +++ b/services/web/frontend/js/features/symbol-palette/components/symbol-palette-content.js @@ -3,7 +3,6 @@ import { useCallback, useEffect, useMemo, useRef, useState } from 'react' import { useTranslation } from 'react-i18next' import PropTypes from 'prop-types' import { matchSorter } from 'match-sorter' - import symbols from '../data/symbols.json' import { buildCategorisedSymbols, createCategories } from '../utils/categories' import SymbolPaletteSearch from './symbol-palette-search' @@ -67,11 +66,11 @@ export default function SymbolPaletteContent({ handleSelect }) {
-
+
+
-
+
{symbol.description}
-
{symbol.command}
+
+ {symbol.command} +
{symbol.notes && ( -
{symbol.notes}
+
+ {symbol.notes} +
)} - +
} + overlayProps={{ placement: 'top', trigger: ['hover', 'focus'] }} > - + ) } + SymbolPaletteItem.propTypes = { symbol: PropTypes.shape({ codepoint: PropTypes.string.isRequired, diff --git a/services/web/frontend/js/features/symbol-palette/components/symbol-palette-search.js b/services/web/frontend/js/features/symbol-palette/components/symbol-palette-search.js index cf5a1eb2a75..7d52a828746 100644 --- a/services/web/frontend/js/features/symbol-palette/components/symbol-palette-search.js +++ b/services/web/frontend/js/features/symbol-palette/components/symbol-palette-search.js @@ -1,7 +1,7 @@ import { useCallback, useEffect, useState } from 'react' import { useTranslation } from 'react-i18next' import PropTypes from 'prop-types' -import { FormControl } from 'react-bootstrap' +import OLFormControl from '@/features/ui/components/ol/ol-form-control' import useDebounce from '../../../shared/hooks/use-debounce' export default function SymbolPaletteSearch({ setInput, inputRef }) { @@ -24,10 +24,10 @@ export default function SymbolPaletteSearch({ setInput, inputRef }) { ) return ( - ) } + SymbolPaletteSearch.propTypes = { setInput: PropTypes.func.isRequired, inputRef: PropTypes.object.isRequired, -} +}; diff --git a/services/web/frontend/stylesheets/modules/symbol-palette.scss b/services/web/frontend/stylesheets/modules/symbol-palette.scss index 9a6aa7e0319..2841b374e1a 100644 --- a/services/web/frontend/stylesheets/modules/symbol-palette.scss +++ b/services/web/frontend/stylesheets/modules/symbol-palette.scss @@ -154,6 +154,8 @@ .symbol-palette-close-button-outer { display: flex; + align-items: center; + margin-right: var(--spacing-01); } .symbol-palette-close-button { From 03bc6b4bde21d926fa331bba796c4565b0ec6cef Mon Sep 17 00:00:00 2001 From: yu-i-i Date: Fri, 4 Apr 2025 15:14:14 +0200 Subject: [PATCH 017/106] Allow EXTERNAL_AUTH to be undefined, fixes #26 --- services/web/modules/authentication/ldap/index.mjs | 2 +- services/web/modules/authentication/oidc/index.mjs | 2 +- services/web/modules/authentication/saml/index.mjs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/services/web/modules/authentication/ldap/index.mjs b/services/web/modules/authentication/ldap/index.mjs index 244a8db8e72..94743a66117 100644 --- a/services/web/modules/authentication/ldap/index.mjs +++ b/services/web/modules/authentication/ldap/index.mjs @@ -1,5 +1,5 @@ let ldapModule = {} -if (process.env.EXTERNAL_AUTH.includes('ldap')) { +if (process.env.EXTERNAL_AUTH?.includes('ldap')) { const { default: LDAPModuleManager } = await import('./app/src/LDAPModuleManager.mjs') const { default: router } = await import('./app/src/LDAPRouter.mjs') LDAPModuleManager.initSettings() diff --git a/services/web/modules/authentication/oidc/index.mjs b/services/web/modules/authentication/oidc/index.mjs index 51d9e0d483d..f10ff64c822 100644 --- a/services/web/modules/authentication/oidc/index.mjs +++ b/services/web/modules/authentication/oidc/index.mjs @@ -1,5 +1,5 @@ let oidcModule = {} -if (process.env.EXTERNAL_AUTH.includes('oidc')) { +if (process.env.EXTERNAL_AUTH?.includes('oidc')) { const { default: OIDCModuleManager } = await import('./app/src/OIDCModuleManager.mjs') const { default: router } = await import('./app/src/OIDCRouter.mjs') OIDCModuleManager.initSettings() diff --git a/services/web/modules/authentication/saml/index.mjs b/services/web/modules/authentication/saml/index.mjs index 2d6ee5706c6..36f0281637a 100644 --- a/services/web/modules/authentication/saml/index.mjs +++ b/services/web/modules/authentication/saml/index.mjs @@ -1,5 +1,5 @@ let samlModule = {} -if (process.env.EXTERNAL_AUTH.includes('saml')) { +if (process.env.EXTERNAL_AUTH?.includes('saml')) { const { default: SAMLModuleManager } = await import('./app/src/SAMLModuleManager.mjs') const { default: router } = await import('./app/src/SAMLRouter.mjs') const { default: nonCsrfRouter } = await import('./app/src/SAMLNonCsrfRouter.mjs') From a66438d4d27b183ba7acfdcbbe70b2f5530aa9c4 Mon Sep 17 00:00:00 2001 From: yu-i-i Date: Tue, 8 Apr 2025 16:49:58 +0200 Subject: [PATCH 018/106] Track changes / comments: update backend to support frontend changes --- .../hooks/use-review-panel-state.ts | 1659 ----------------- .../app/src/TrackChangesController.js | 61 +- 2 files changed, 22 insertions(+), 1698 deletions(-) delete mode 100644 services/web/frontend/js/features/ide-react/context/review-panel/hooks/use-review-panel-state.ts diff --git a/services/web/frontend/js/features/ide-react/context/review-panel/hooks/use-review-panel-state.ts b/services/web/frontend/js/features/ide-react/context/review-panel/hooks/use-review-panel-state.ts deleted file mode 100644 index c712384163f..00000000000 --- a/services/web/frontend/js/features/ide-react/context/review-panel/hooks/use-review-panel-state.ts +++ /dev/null @@ -1,1659 +0,0 @@ -import { useState, useEffect, useMemo, useCallback, useRef } from 'react' -import { useTranslation } from 'react-i18next' -import { isEqual, cloneDeep } from 'lodash' -import usePersistedState from '@/shared/hooks/use-persisted-state' -import useScopeValue from '../../../../../shared/hooks/use-scope-value' -import useSocketListener from '@/features/ide-react/hooks/use-socket-listener' -import useAbortController from '@/shared/hooks/use-abort-controller' -import useScopeEventEmitter from '@/shared/hooks/use-scope-event-emitter' -import useLayoutToLeft from '@/features/ide-react/context/review-panel/hooks/useLayoutToLeft' -import { sendMB } from '@/infrastructure/event-tracking' -import { - dispatchReviewPanelLayout as handleLayoutChange, - UpdateType, -} from '@/features/source-editor/extensions/changes/change-manager' -import { useProjectContext } from '@/shared/context/project-context' -import { useLayoutContext } from '@/shared/context/layout-context' -import { useUserContext } from '@/shared/context/user-context' -import { useIdeReactContext } from '@/features/ide-react/context/ide-react-context' -import { useConnectionContext } from '@/features/ide-react/context/connection-context' -import { usePermissionsContext } from '@/features/ide-react/context/permissions-context' -import { useModalsContext } from '@/features/ide-react/context/modals-context' -import { - EditorManager, - useEditorManagerContext, -} from '@/features/ide-react/context/editor-manager-context' -import { debugConsole } from '@/utils/debugging' -import { deleteJSON, getJSON, postJSON } from '@/infrastructure/fetch-json' -import RangesTracker from '@overleaf/ranges-tracker' -import type * as ReviewPanel from '@/features/source-editor/context/review-panel/types/review-panel-state' -import { - CommentId, - ReviewPanelCommentThreadMessage, - ReviewPanelCommentThreads, - ReviewPanelDocEntries, - SubView, - ThreadId, -} from '../../../../../../../types/review-panel/review-panel' -import { UserId } from '../../../../../../../types/user' -import { PublicAccessLevel } from '../../../../../../../types/public-access-level' -import { - DeepReadonly, - Entries, - MergeAndOverride, -} from '../../../../../../../types/utils' -import { ReviewPanelCommentThread } from '../../../../../../../types/review-panel/comment-thread' -import { DocId } from '../../../../../../../types/project-settings' -import { - ReviewPanelAddCommentEntry, - ReviewPanelAggregateChangeEntry, - ReviewPanelBulkActionsEntry, - ReviewPanelChangeEntry, - ReviewPanelCommentEntry, - ReviewPanelEntry, -} from '../../../../../../../types/review-panel/entry' -import { - ReviewPanelCommentThreadMessageApi, - ReviewPanelCommentThreadsApi, -} from '../../../../../../../types/review-panel/api' -import { DateString } from '../../../../../../../types/helpers/date' -import { - Change, - CommentOperation, - EditOperation, -} from '../../../../../../../types/change' -import { RangesTrackerWithResolvedThreadIds } from '@/features/ide-react/editor/document-container' -import getMeta from '@/utils/meta' -import { useEditorContext } from '@/shared/context/editor-context' -import { getHueForUserId } from '@/shared/utils/colors' - -const dispatchReviewPanelEvent = (type: string, payload?: any) => { - window.dispatchEvent( - new CustomEvent('review-panel:event', { - detail: { type, payload }, - }) - ) -} - -const formatUser = (user: any): any => { - let isSelf, name - const id = - (user != null ? user._id : undefined) || - (user != null ? user.id : undefined) - - if (id == null) { - return { - id: 'anonymous-user', - email: null, - name: 'Anonymous', - isSelf: false, - hue: getHueForUserId(), - avatar_text: 'A', - } - } - if (id === getMeta('ol-user_id')) { - name = 'You' - isSelf = true - } else { - name = [user.first_name, user.last_name] - .filter(n => n != null && n !== '') - .join(' ') - if (name === '') { - name = - (user.email != null ? user.email.split('@')[0] : undefined) || 'Unknown' - } - isSelf = false - } - return { - id, - email: user.email, - name, - isSelf, - hue: getHueForUserId(id), - avatar_text: [user.first_name, user.last_name] - .filter(n => n != null) - .map(n => n[0]) - .join(''), - } -} - -const formatComment = ( - comment: ReviewPanelCommentThreadMessageApi -): ReviewPanelCommentThreadMessage => { - const commentTyped = comment as unknown as ReviewPanelCommentThreadMessage - commentTyped.user = formatUser(comment.user) - commentTyped.timestamp = new Date(comment.timestamp) - return commentTyped -} - -function useReviewPanelState(): ReviewPanel.ReviewPanelState { - const { t } = useTranslation() - const { reviewPanelOpen, setReviewPanelOpen, setMiniReviewPanelVisible } = - useLayoutContext() - const { projectId } = useIdeReactContext() - const project = useProjectContext() - const user = useUserContext() - const { socket } = useConnectionContext() - const { - features: { trackChangesVisible, trackChanges }, - } = project - const { isRestrictedTokenMember } = useEditorContext() - const { - openDocWithId, - currentDocument, - currentDocumentId, - wantTrackChanges, - setWantTrackChanges, - } = useEditorManagerContext() as MergeAndOverride< - EditorManager, - { currentDocumentId: DocId } - > - // TODO permissions to be removed from the review panel context. It currently acts just as a proxy. - const permissions = usePermissionsContext() - const { showGenericMessageModal } = useModalsContext() - const addCommentEmitter = useScopeEventEmitter('comment:start_adding') - - const layoutToLeft = useLayoutToLeft('.ide-react-editor-panel') - const [subView, setSubView] = - useState>('cur_file') - const [isOverviewLoading, setIsOverviewLoading] = - useState>(false) - // All selected changes. If an aggregated change (insertion + deletion) is selected, the two ids - // will be present. The length of this array will differ from the count below (see explanation). - const selectedEntryIds = useRef([]) - // A count of user-facing selected changes. An aggregated change (insertion + deletion) will count - // as only one. - const [nVisibleSelectedChanges, setNVisibleSelectedChanges] = - useState>(0) - const [collapsed, setCollapsed] = usePersistedState< - ReviewPanel.Value<'collapsed'> - >(`docs_collapsed_state:${projectId}`, {}, false, true) - const [commentThreads, setCommentThreads] = useState< - ReviewPanel.Value<'commentThreads'> - >({}) - const [entries, setEntries] = useState>({}) - const [users, setUsers] = useScopeValue>('users') - const [resolvedComments, setResolvedComments] = useState< - ReviewPanel.Value<'resolvedComments'> - >({}) - - const [shouldCollapse, setShouldCollapse] = - useState>(true) - const [lineHeight, setLineHeight] = - useState>(0) - - const [formattedProjectMembers, setFormattedProjectMembers] = useState< - ReviewPanel.Value<'formattedProjectMembers'> - >({}) - const [trackChangesState, setTrackChangesState] = useState< - ReviewPanel.Value<'trackChangesState'> - >({}) - const [trackChangesOnForEveryone, setTrackChangesOnForEveryone] = - useState>(false) - const [trackChangesOnForGuests, setTrackChangesOnForGuests] = - useState>(false) - const [trackChangesForGuestsAvailable, setTrackChangesForGuestsAvailable] = - useState>(false) - - const [resolvedThreadIds, setResolvedThreadIds] = useState< - Record - >({}) - - const [loadingThreads, setLoadingThreads] = - useScopeValue('loadingThreads') - - const loadThreadsController = useAbortController() - const threadsLoadedOnceRef = useRef(false) - const loadingThreadsInProgressRef = useRef(false) - const ensureThreadsAreLoaded = useCallback(() => { - if (threadsLoadedOnceRef.current) { - // We get any updates in real time so only need to load them once. - return - } - threadsLoadedOnceRef.current = true - loadingThreadsInProgressRef.current = true - - return getJSON(`/project/${projectId}/threads`, { - signal: loadThreadsController.signal, - }) - .then(threads => { - setLoadingThreads(false) - const tempResolvedThreadIds: typeof resolvedThreadIds = {} - const threadsEntries = Object.entries(threads) as [ - [ - ThreadId, - MergeAndOverride< - ReviewPanelCommentThread, - ReviewPanelCommentThreadsApi[ThreadId] - >, - ], - ] - for (const [threadId, thread] of threadsEntries) { - for (const comment of thread.messages) { - formatComment(comment) - } - if (thread.resolved_by_user) { - thread.resolved_by_user = formatUser(thread.resolved_by_user) - tempResolvedThreadIds[threadId] = true - } - } - setResolvedThreadIds(tempResolvedThreadIds) - setCommentThreads(threads as unknown as ReviewPanelCommentThreads) - - dispatchReviewPanelEvent('loaded_threads') - handleLayoutChange({ async: true }) - - return { - resolvedThreadIds: tempResolvedThreadIds, - commentThreads: threads, - } - }) - .catch(debugConsole.error) - .finally(() => { - loadingThreadsInProgressRef.current = false - }) - }, [loadThreadsController.signal, projectId, setLoadingThreads]) - - const rangesTrackers = useRef< - Record - >({}) - const refreshingRangeUsers = useRef(false) - const refreshedForUserIds = useRef(new Set()) - const refreshChangeUsers = useCallback( - (userId: UserId | null) => { - if (userId != null) { - if (refreshedForUserIds.current.has(userId)) { - // We've already tried to refresh to get this user id, so stop it looping - return - } - refreshedForUserIds.current.add(userId) - } - - // Only do one refresh at once - if (refreshingRangeUsers.current) { - return - } - refreshingRangeUsers.current = true - - getJSON(`/project/${projectId}/changes/users`) - .then(usersResponse => { - refreshingRangeUsers.current = false - const tempUsers = {} as ReviewPanel.Value<'users'> - // Always include ourself, since if we submit an op, we might need to display info - // about it locally before it has been flushed through the server - if (user) { - if (user.id) { - tempUsers[user.id] = formatUser(user) - } else { - tempUsers['anonymous-user'] = formatUser(user) - } - } - for (const user of usersResponse) { - if (user.id) { - tempUsers[user.id] = formatUser(user) - } else { - tempUsers['anonymous-user'] = formatUser(user) - } - } - setUsers(tempUsers) - }) - .catch(error => { - refreshingRangeUsers.current = false - debugConsole.error(error) - }) - }, - [projectId, setUsers, user] - ) - - const getChangeTracker = useCallback( - (docId: DocId) => { - if (!rangesTrackers.current[docId]) { - const rangesTracker = new RangesTracker([], []) - ;( - rangesTracker as RangesTrackerWithResolvedThreadIds - ).resolvedThreadIds = { ...resolvedThreadIds } - rangesTrackers.current[docId] = - rangesTracker as RangesTrackerWithResolvedThreadIds - } - return rangesTrackers.current[docId]! - }, - [resolvedThreadIds] - ) - - const getDocEntries = useCallback( - (docId: DocId) => { - return entries[docId] ?? ({} as ReviewPanelDocEntries) - }, - [entries] - ) - - const getDocResolvedComments = useCallback( - (docId: DocId) => { - return resolvedComments[docId] ?? ({} as ReviewPanelDocEntries) - }, - [resolvedComments] - ) - - const getThread = useCallback( - (threadId: ThreadId) => { - return ( - commentThreads[threadId] ?? - ({ messages: [] } as ReviewPanelCommentThread) - ) - }, - [commentThreads] - ) - - const updateEntries = useCallback( - async (docId: DocId) => { - const rangesTracker = getChangeTracker(docId) - const docEntries = cloneDeep(getDocEntries(docId)) - const docResolvedComments = cloneDeep(getDocResolvedComments(docId)) - // Assume we'll delete everything until we see it, then we'll remove it from this object - const deleteChanges = new Set() - - for (const [id, change] of Object.entries(docEntries) as Entries< - typeof docEntries - >) { - if ( - 'entry_ids' in change && - id !== 'add-comment' && - id !== 'bulk-actions' - ) { - for (const entryId of change.entry_ids) { - deleteChanges.add(entryId) - } - } - } - for (const [, change] of Object.entries(docResolvedComments) as Entries< - typeof docResolvedComments - >) { - if ('entry_ids' in change) { - for (const entryId of change.entry_ids) { - deleteChanges.add(entryId) - } - } - } - - let potentialAggregate = false - let prevInsertion = null - - for (const change of rangesTracker.changes as any[]) { - if ( - potentialAggregate && - change.op.d && - change.op.p === prevInsertion.op.p + prevInsertion.op.i.length && - change.metadata.user_id === prevInsertion.metadata.user_id - ) { - // An actual aggregate op. - const aggregateChangeEntries = docEntries as Record< - string, - ReviewPanelAggregateChangeEntry - > - aggregateChangeEntries[prevInsertion.id].type = 'aggregate-change' - aggregateChangeEntries[prevInsertion.id].metadata.replaced_content = - change.op.d - aggregateChangeEntries[prevInsertion.id].entry_ids.push(change.id) - } else { - if (docEntries[change.id] == null) { - docEntries[change.id] = {} as ReviewPanelEntry - } - deleteChanges.delete(change.id) - const newEntry: Partial = { - type: change.op.i ? 'insert' : 'delete', - entry_ids: [change.id], - content: change.op.i || change.op.d, - offset: change.op.p, - metadata: change.metadata, - } - for (const [key, value] of Object.entries(newEntry) as Entries< - typeof newEntry - >) { - const entriesTyped = docEntries[change.id] as Record - entriesTyped[key] = value - } - } - - if (change.op.i) { - potentialAggregate = true - prevInsertion = change - } else { - potentialAggregate = false - prevInsertion = null - } - - if (!users[change.metadata.user_id]) { - if (!isRestrictedTokenMember) { - refreshChangeUsers(change.metadata.user_id) - } - } - } - - let localResolvedThreadIds = resolvedThreadIds - - if (!isRestrictedTokenMember && rangesTracker.comments.length > 0) { - const threadsLoadResult = await ensureThreadsAreLoaded() - if (threadsLoadResult?.resolvedThreadIds) { - localResolvedThreadIds = threadsLoadResult.resolvedThreadIds - } - } else if (loadingThreads) { - // ensure that tracked changes are highlighted even if no comments are loaded - setLoadingThreads(false) - dispatchReviewPanelEvent('loaded_threads') - } - - if (!loadingThreadsInProgressRef.current) { - for (const comment of rangesTracker.comments) { - const commentId = comment.id as ThreadId - deleteChanges.delete(commentId) - - let newComment: any - if (localResolvedThreadIds[comment.op.t]) { - docResolvedComments[commentId] ??= {} as ReviewPanelCommentEntry - newComment = docResolvedComments[commentId] - delete docEntries[commentId] - } else { - docEntries[commentId] ??= {} as ReviewPanelEntry - newComment = docEntries[commentId] - delete docResolvedComments[commentId] - } - - newComment.type = 'comment' - newComment.thread_id = comment.op.t - newComment.entry_ids = [comment.id] - newComment.content = comment.op.c - newComment.offset = comment.op.p - } - } - - deleteChanges.forEach(changeId => { - delete docEntries[changeId] - delete docResolvedComments[changeId] - }) - - setEntries(prev => { - return isEqual(prev[docId], docEntries) - ? prev - : { ...prev, [docId]: docEntries } - }) - setResolvedComments(prev => { - return isEqual(prev[docId], docResolvedComments) - ? prev - : { ...prev, [docId]: docResolvedComments } - }) - - return docEntries - }, - [ - getChangeTracker, - getDocEntries, - getDocResolvedComments, - refreshChangeUsers, - resolvedThreadIds, - users, - ensureThreadsAreLoaded, - loadingThreads, - setLoadingThreads, - isRestrictedTokenMember, - ] - ) - - const regenerateTrackChangesId = useCallback( - (doc: typeof currentDocument) => { - if (doc) { - const currentChangeTracker = getChangeTracker(doc.doc_id as DocId) - const oldId = currentChangeTracker.getIdSeed() - const newId = RangesTracker.generateIdSeed() - currentChangeTracker.setIdSeed(newId) - doc.setTrackChangesIdSeeds({ pending: newId, inflight: oldId }) - } - }, - [getChangeTracker] - ) - - useEffect(() => { - if (!currentDocument) { - return - } - // The open doc range tracker is kept up to date in real-time so - // replace any outdated info with this - const rangesTracker = currentDocument.ranges! - ;(rangesTracker as RangesTrackerWithResolvedThreadIds).resolvedThreadIds = { - ...resolvedThreadIds, - } - rangesTrackers.current[currentDocument.doc_id as DocId] = - rangesTracker as RangesTrackerWithResolvedThreadIds - currentDocument.on('flipped_pending_to_inflight', () => - regenerateTrackChangesId(currentDocument) - ) - regenerateTrackChangesId(currentDocument) - - return () => { - currentDocument.off('flipped_pending_to_inflight') - } - }, [currentDocument, regenerateTrackChangesId, resolvedThreadIds]) - - const currentUserType = useCallback((): 'member' | 'guest' | 'anonymous-user' => { - if (!user) { - return 'anonymous-user' - } - if (project.owner._id === user.id) { - return 'member' - } - for (const member of project.members as any[]) { - if (member._id === user.id) { - return 'member' - } - } - return 'guest' - }, [project.members, project.owner, user]) - - const applyClientTrackChangesStateToServer = useCallback( - ( - trackChangesOnForEveryone: boolean, - trackChangesOnForGuests: boolean, - trackChangesState: ReviewPanel.Value<'trackChangesState'> - ) => { - const data: { - on?: boolean - on_for?: Record - on_for_guests?: boolean - } = {} - if (trackChangesOnForEveryone) { - data.on = true - } else { - data.on_for = {} - const entries = Object.entries(trackChangesState) as Array< - [ - UserId, - NonNullable< - (typeof trackChangesState)[keyof typeof trackChangesState] - >, - ] - > - for (const [userId, { value }] of entries) { - data.on_for[userId] = value - } - if (trackChangesOnForGuests) { - data.on_for_guests = true - } - } - postJSON(`/project/${projectId}/track_changes`, { - body: data, - }).catch(debugConsole.error) - }, - [projectId] - ) - - const setGuestsTCState = useCallback( - (newValue: boolean) => { - setTrackChangesOnForGuests(newValue) - if (currentUserType() === 'guest' || currentUserType() === 'anonymous-user') { - setWantTrackChanges(newValue) - } - }, - [currentUserType, setWantTrackChanges] - ) - - const setUserTCState = useCallback( - ( - trackChangesState: DeepReadonly>, - userId: UserId, - newValue: boolean, - isLocal = false - ) => { - const newTrackChangesState: ReviewPanel.Value<'trackChangesState'> = { - ...trackChangesState, - } - const state = - newTrackChangesState[userId] ?? - ({} as NonNullable<(typeof newTrackChangesState)[UserId]>) - newTrackChangesState[userId] = state - - if (state.syncState == null || state.syncState === 'synced') { - state.value = newValue - state.syncState = 'synced' - } else if (state.syncState === 'pending' && state.value === newValue) { - state.syncState = 'synced' - } else if (isLocal) { - state.value = newValue - state.syncState = 'pending' - } - - setTrackChangesState(newTrackChangesState) - - if (userId === user.id) { - setWantTrackChanges(newValue) - } - - return newTrackChangesState - }, - [setWantTrackChanges, user.id] - ) - - const setEveryoneTCState = useCallback( - (newValue: boolean, isLocal = false) => { - setTrackChangesOnForEveryone(newValue) - let newTrackChangesState: ReviewPanel.Value<'trackChangesState'> = { - ...trackChangesState, - } - for (const member of project.members as any[]) { - newTrackChangesState = setUserTCState( - newTrackChangesState, - member._id, - newValue, - isLocal - ) - } - setGuestsTCState(newValue) - - newTrackChangesState = setUserTCState( - newTrackChangesState, - project.owner._id, - newValue, - isLocal - ) - - return { trackChangesState: newTrackChangesState } - }, - [ - project.members, - project.owner._id, - setGuestsTCState, - setUserTCState, - trackChangesState, - ] - ) - - const toggleTrackChangesForEveryone = useCallback< - ReviewPanel.UpdaterFn<'toggleTrackChangesForEveryone'> - >( - (onForEveryone: boolean) => { - const { trackChangesState } = setEveryoneTCState(onForEveryone, true) - setGuestsTCState(onForEveryone) - applyClientTrackChangesStateToServer( - onForEveryone, - onForEveryone, - trackChangesState - ) - }, - [applyClientTrackChangesStateToServer, setEveryoneTCState, setGuestsTCState] - ) - - const toggleTrackChangesForGuests = useCallback< - ReviewPanel.UpdaterFn<'toggleTrackChangesForGuests'> - >( - (onForGuests: boolean) => { - setGuestsTCState(onForGuests) - applyClientTrackChangesStateToServer( - trackChangesOnForEveryone, - onForGuests, - trackChangesState - ) - }, - [ - applyClientTrackChangesStateToServer, - setGuestsTCState, - trackChangesOnForEveryone, - trackChangesState, - ] - ) - - const toggleTrackChangesForUser = useCallback< - ReviewPanel.UpdaterFn<'toggleTrackChangesForUser'> - >( - (onForUser: boolean, userId: UserId) => { - const newTrackChangesState = setUserTCState( - trackChangesState, - userId, - onForUser, - true - ) - applyClientTrackChangesStateToServer( - trackChangesOnForEveryone, - trackChangesOnForGuests, - newTrackChangesState - ) - }, - [ - applyClientTrackChangesStateToServer, - setUserTCState, - trackChangesOnForEveryone, - trackChangesOnForGuests, - trackChangesState, - ] - ) - - const applyTrackChangesStateToClient = useCallback( - (state: boolean | Record) => { - if (typeof state === 'boolean') { - setEveryoneTCState(state) - setGuestsTCState(state) - } else { - setTrackChangesOnForEveryone(false) - // TODO - // @ts-ignore - setGuestsTCState(state.__guests__ === true) - - let newTrackChangesState: ReviewPanel.Value<'trackChangesState'> = { - ...trackChangesState, - } - for (const member of project.members as any[]) { - newTrackChangesState = setUserTCState( - newTrackChangesState, - member._id, - !!state[member._id] - ) - } - newTrackChangesState = setUserTCState( - newTrackChangesState, - project.owner._id, - !!state[project.owner._id] - ) - return newTrackChangesState - } - }, - [ - project.members, - project.owner._id, - setEveryoneTCState, - setGuestsTCState, - setUserTCState, - trackChangesState, - ] - ) - - const setGuestFeatureBasedOnProjectAccessLevel = ( - projectPublicAccessLevel?: PublicAccessLevel - ) => { - setTrackChangesForGuestsAvailable(projectPublicAccessLevel === 'tokenBased') - } - - useEffect(() => { - setGuestFeatureBasedOnProjectAccessLevel(project.publicAccessLevel) - }, [project.publicAccessLevel]) - - useEffect(() => { - if ( - trackChangesForGuestsAvailable || - !trackChangesOnForGuests || - trackChangesOnForEveryone - ) { - return - } - - // Overrides guest setting - toggleTrackChangesForGuests(false) - }, [ - toggleTrackChangesForGuests, - trackChangesForGuestsAvailable, - trackChangesOnForEveryone, - trackChangesOnForGuests, - ]) - - const projectJoinedEffectExecuted = useRef(false) - useEffect(() => { - if (!projectJoinedEffectExecuted.current) { - projectJoinedEffectExecuted.current = true - requestAnimationFrame(() => { - if (trackChanges) { - applyTrackChangesStateToClient(project.trackChangesState) - } else { - applyTrackChangesStateToClient(false) - } - setGuestFeatureBasedOnProjectAccessLevel(project.publicAccessLevel) - }) - } - }, [ - applyTrackChangesStateToClient, - trackChanges, - project.publicAccessLevel, - project.trackChangesState, - ]) - - useEffect(() => { - setFormattedProjectMembers(prevState => { - const tempFormattedProjectMembers: typeof prevState = {} - if (project.owner) { - tempFormattedProjectMembers[project.owner._id] = formatUser( - project.owner - ) - } - const members = project.members ?? [] - for (const member of members) { - if (member.privileges === 'readAndWrite') { - if (!trackChangesState[member._id]) { - // An added member will have track changes enabled if track changes is on for everyone - setUserTCState( - trackChangesState, - member._id, - trackChangesOnForEveryone, - true - ) - } - tempFormattedProjectMembers[member._id] = formatUser(member) - } - } - return tempFormattedProjectMembers - }) - }, [ - project.members, - project.owner, - setUserTCState, - trackChangesOnForEveryone, - trackChangesState, - ]) - - useSocketListener( - socket, - 'toggle-track-changes', - applyTrackChangesStateToClient - ) - - const gotoEntry = useCallback( - (docId: DocId, entryOffset: number) => { - openDocWithId(docId, { gotoOffset: entryOffset }) - }, - [openDocWithId] - ) - - const view = reviewPanelOpen ? subView : 'mini' - - const toggleReviewPanel = useCallback(() => { - if (!trackChangesVisible) { - return - } - setReviewPanelOpen(!reviewPanelOpen) - sendMB('rp-toggle-panel', { - value: !reviewPanelOpen, - }) - }, [reviewPanelOpen, setReviewPanelOpen, trackChangesVisible]) - - const onCommentResolved = useCallback( - (threadId: ThreadId, user: any) => { - setCommentThreads(prevState => { - const thread = { ...getThread(threadId) } - thread.resolved = true - thread.resolved_by_user = formatUser(user) - thread.resolved_at = new Date().toISOString() as DateString - return { ...prevState, [threadId]: thread } - }) - setResolvedThreadIds(prevState => ({ ...prevState, [threadId]: true })) - setTimeout(() => { - dispatchReviewPanelEvent('comment:resolve_threads', [threadId]) - }) - }, - [getThread] - ) - - const resolveComment = useCallback( - (docId: DocId, entryId: ThreadId) => { - const docEntries = getDocEntries(docId) - const entry = docEntries[entryId] as ReviewPanelCommentEntry - - setEntries(prevState => ({ - ...prevState, - [docId]: { - ...prevState[docId], - [entryId]: { - ...prevState[docId][entryId], - focused: false, - }, - }, - })) - - postJSON( - `/project/${projectId}/doc/${docId}/thread/${entry.thread_id}/resolve` - ) - onCommentResolved(entry.thread_id, user) - sendMB('rp-comment-resolve', { view }) - }, - [getDocEntries, onCommentResolved, projectId, user, view] - ) - - const onCommentReopened = useCallback( - (threadId: ThreadId) => { - setCommentThreads(prevState => { - const { - resolved: _1, - resolved_by_user: _2, - resolved_at: _3, - ...thread - } = getThread(threadId) - return { ...prevState, [threadId]: thread } - }) - setResolvedThreadIds(({ [threadId]: _, ...resolvedThreadIds }) => { - return resolvedThreadIds - }) - setTimeout(() => { - dispatchReviewPanelEvent('comment:unresolve_thread', threadId) - }) - }, - [getThread] - ) - - const unresolveComment = useCallback( - (docId: DocId, threadId: ThreadId) => { - onCommentReopened(threadId) - const url = `/project/${projectId}/doc/${docId}/thread/${threadId}/reopen` - postJSON(url).catch(debugConsole.error) - sendMB('rp-comment-reopen') - }, - [onCommentReopened, projectId] - ) - - const onThreadDeleted = useCallback((threadId: ThreadId) => { - setResolvedThreadIds(({ [threadId]: _, ...resolvedThreadIds }) => { - return resolvedThreadIds - }) - setCommentThreads(({ [threadId]: _, ...commentThreads }) => { - return commentThreads - }) - dispatchReviewPanelEvent('comment:remove', threadId) - }, []) - - const deleteThread = useCallback( - (docId: DocId, threadId: ThreadId) => { - onThreadDeleted(threadId) - deleteJSON(`/project/${projectId}/doc/${docId}/thread/${threadId}`).catch( - debugConsole.error - ) - sendMB('rp-comment-delete') - }, - [onThreadDeleted, projectId] - ) - - const onCommentEdited = useCallback( - (threadId: ThreadId, commentId: CommentId, content: string) => { - setCommentThreads(prevState => { - const thread = { ...getThread(threadId) } - thread.messages = thread.messages.map(message => { - return message.id === commentId ? { ...message, content } : message - }) - return { ...prevState, [threadId]: thread } - }) - }, - [getThread] - ) - - const saveEdit = useCallback( - (threadId: ThreadId, commentId: CommentId, content: string) => { - const url = `/project/${projectId}/thread/${threadId}/messages/${commentId}/edit` - postJSON(url, { body: { content } }).catch(debugConsole.error) - handleLayoutChange({ async: true }) - }, - [projectId] - ) - - const onCommentDeleted = useCallback( - (threadId: ThreadId, commentId: CommentId) => { - setCommentThreads(prevState => { - const thread = { ...getThread(threadId) } - thread.messages = thread.messages.filter(m => m.id !== commentId) - return { ...prevState, [threadId]: thread } - }) - }, - [getThread] - ) - - const deleteComment = useCallback( - (threadId: ThreadId, commentId: CommentId) => { - onCommentDeleted(threadId, commentId) - deleteJSON( - `/project/${projectId}/thread/${threadId}/messages/${commentId}` - ).catch(debugConsole.error) - handleLayoutChange({ async: true }) - }, - [onCommentDeleted, projectId] - ) - - const doAcceptChanges = useCallback( - (entryIds: ThreadId[]) => { - const url = `/project/${projectId}/doc/${currentDocumentId}/changes/accept` - postJSON(url, { body: { change_ids: entryIds } }).catch( - debugConsole.error - ) - dispatchReviewPanelEvent('changes:accept', entryIds) - }, - [currentDocumentId, projectId] - ) - - const acceptChanges = useCallback( - (entryIds: ThreadId[]) => { - doAcceptChanges(entryIds) - sendMB('rp-changes-accepted', { view }) - }, - [doAcceptChanges, view] - ) - - const doRejectChanges = useCallback((entryIds: ThreadId[]) => { - dispatchReviewPanelEvent('changes:reject', entryIds) - }, []) - - const rejectChanges = useCallback( - (entryIds: ThreadId[]) => { - doRejectChanges(entryIds) - sendMB('rp-changes-rejected', { view }) - }, - [doRejectChanges, view] - ) - - const bulkAcceptActions = useCallback(() => { - doAcceptChanges(selectedEntryIds.current) - sendMB('rp-bulk-accept', { view, nEntries: nVisibleSelectedChanges }) - }, [doAcceptChanges, nVisibleSelectedChanges, view]) - - const bulkRejectActions = useCallback(() => { - doRejectChanges(selectedEntryIds.current) - sendMB('rp-bulk-reject', { view, nEntries: nVisibleSelectedChanges }) - }, [doRejectChanges, nVisibleSelectedChanges, view]) - - const refreshRanges = useCallback(() => { - type Doc = { - id: DocId - ranges: { - comments?: Change[] - changes?: Change[] - } - } - - return getJSON(`/project/${projectId}/ranges`) - .then(docs => { - setCollapsed(prevState => { - const collapsed = { ...prevState } - docs.forEach(doc => { - if (collapsed[doc.id] == null) { - collapsed[doc.id] = false - } - }) - return collapsed - }) - - docs.forEach(async doc => { - if (doc.id !== currentDocumentId) { - // this is kept up to date in real-time, don't overwrite - const rangesTracker = getChangeTracker(doc.id) - rangesTracker.comments = doc.ranges?.comments ?? [] - rangesTracker.changes = doc.ranges?.changes ?? [] - } - }) - - return Promise.all(docs.map(doc => updateEntries(doc.id))) - }) - .catch(debugConsole.error) - }, [ - currentDocumentId, - getChangeTracker, - projectId, - setCollapsed, - updateEntries, - ]) - - const handleSetSubview = useCallback((subView: SubView) => { - setSubView(subView) - sendMB('rp-subview-change', { subView }) - }, []) - - const submitReply = useCallback( - (threadId: ThreadId, replyContent: string) => { - const url = `/project/${projectId}/thread/${threadId}/messages` - postJSON(url, { body: { content: replyContent } }).catch(() => { - showGenericMessageModal( - t('error_submitting_comment'), - t('comment_submit_error') - ) - }) - - const trackingMetadata = { - view, - size: replyContent.length, - thread: threadId, - } - - setCommentThreads(prevState => ({ - ...prevState, - [threadId]: { ...getThread(threadId), submitting: true }, - })) - handleLayoutChange({ async: true }) - sendMB('rp-comment-reply', trackingMetadata) - }, - [getThread, projectId, showGenericMessageModal, t, view] - ) - - // TODO `submitNewComment` is partially localized in the `add-comment-entry` component. - const submitNewComment = useCallback( - (content: string) => { - if (!content) { - return - } - - const entries = getDocEntries(currentDocumentId) - const addCommentEntry = entries['add-comment'] as - | ReviewPanelAddCommentEntry - | undefined - - if (!addCommentEntry) { - return - } - - const { offset, length } = addCommentEntry - const threadId = RangesTracker.generateId() as ThreadId - setCommentThreads(prevState => ({ - ...prevState, - [threadId]: { ...getThread(threadId), submitting: true }, - })) - - const url = `/project/${projectId}/thread/${threadId}/messages` - postJSON(url, { body: { content } }) - .then(() => { - dispatchReviewPanelEvent('comment:add', { threadId, offset, length }) - handleLayoutChange({ async: true }) - sendMB('rp-new-comment', { size: content.length }) - }) - .catch(() => { - showGenericMessageModal( - t('error_submitting_comment'), - t('comment_submit_error') - ) - }) - }, - [ - currentDocumentId, - getDocEntries, - getThread, - projectId, - showGenericMessageModal, - t, - ] - ) - - const [isAddingComment, setIsAddingComment] = useState(false) - const [navHeight, setNavHeight] = useState(0) - const [toolbarHeight, setToolbarHeight] = useState(0) - const [layoutSuspended, setLayoutSuspended] = useState(false) - const [unsavedComment, setUnsavedComment] = useState('') - - useEffect(() => { - if (!trackChangesVisible) { - setReviewPanelOpen(false) - } - }, [trackChangesVisible, setReviewPanelOpen]) - - const hasEntries = useMemo(() => { - const docEntries = getDocEntries(currentDocumentId) - const permEntriesCount = Object.keys(docEntries).filter(key => { - return !['add-comment', 'bulk-actions'].includes(key) - }).length - return permEntriesCount > 0 && trackChangesVisible - }, [currentDocumentId, getDocEntries, trackChangesVisible]) - - useEffect(() => { - setMiniReviewPanelVisible(!reviewPanelOpen && !!hasEntries) - }, [reviewPanelOpen, hasEntries, setMiniReviewPanelVisible]) - - // listen for events from the CodeMirror 6 track changes extension - useEffect(() => { - const toggleTrackChangesFromKbdShortcut = () => { - const userId = user.id - if (trackChangesVisible && trackChanges && userId) { - const state = trackChangesState[userId] - if (state) { - toggleTrackChangesForUser(!state.value, userId) - } - } - } - - const editorLineHeightChanged = (payload: typeof lineHeight) => { - setLineHeight(payload) - handleLayoutChange() - } - - const editorTrackChangesChanged = async () => { - const tempEntries = cloneDeep(await updateEntries(currentDocumentId)) - - // `tempEntries` would be mutated - dispatchReviewPanelEvent('recalculate-screen-positions', { - entries: tempEntries, - updateType: 'trackedChangesChange', - }) - - // The state should be updated after dispatching the 'recalculate-screen-positions' - // event as `tempEntries` will be mutated - setEntries(prev => ({ ...prev, [currentDocumentId]: tempEntries })) - handleLayoutChange() - } - - const editorTrackChangesVisibilityChanged = () => { - handleLayoutChange({ async: true, animate: false }) - } - - const editorFocusChanged = ( - selectionOffsetStart: number, - selectionOffsetEnd: number, - selection: boolean, - updateType: UpdateType - ) => { - let tempEntries = cloneDeep(getDocEntries(currentDocumentId)) - // All selected changes will be added to this array. - selectedEntryIds.current = [] - // Count of user-visible changes, i.e. an aggregated change will count as one. - let tempNVisibleSelectedChanges = 0 - - const offset = selectionOffsetStart - const length = selectionOffsetEnd - selectionOffsetStart - - // Recreate the add comment and bulk actions entries only when - // necessary. This is to avoid the UI thinking that these entries have - // changed and getting into an infinite loop. - if (selection) { - const existingAddComment = tempEntries[ - 'add-comment' - ] as ReviewPanelAddCommentEntry - if ( - !existingAddComment || - existingAddComment.offset !== offset || - existingAddComment.length !== length - ) { - tempEntries['add-comment'] = { - type: 'add-comment', - offset, - length, - } as ReviewPanelAddCommentEntry - } - const existingBulkActions = tempEntries[ - 'bulk-actions' - ] as ReviewPanelBulkActionsEntry - if ( - !existingBulkActions || - existingBulkActions.offset !== offset || - existingBulkActions.length !== length - ) { - tempEntries['bulk-actions'] = { - type: 'bulk-actions', - offset, - length, - } as ReviewPanelBulkActionsEntry - } - } else { - delete (tempEntries as Partial)['add-comment'] - delete (tempEntries as Partial)['bulk-actions'] - } - - for (const [key, entry] of Object.entries(tempEntries) as Entries< - typeof tempEntries - >) { - let isChangeEntryAndWithinSelection = false - if (entry.type === 'comment' && !resolvedThreadIds[entry.thread_id]) { - tempEntries = { - ...tempEntries, - [key]: { - ...tempEntries[key], - focused: - entry.offset <= selectionOffsetStart && - selectionOffsetStart <= entry.offset + entry.content.length, - }, - } - } else if ( - entry.type === 'insert' || - entry.type === 'aggregate-change' - ) { - isChangeEntryAndWithinSelection = - entry.offset >= selectionOffsetStart && - entry.offset + entry.content.length <= selectionOffsetEnd - tempEntries = { - ...tempEntries, - [key]: { - ...tempEntries[key], - focused: - entry.offset <= selectionOffsetStart && - selectionOffsetStart <= entry.offset + entry.content.length, - }, - } - } else if (entry.type === 'delete') { - isChangeEntryAndWithinSelection = - selectionOffsetStart <= entry.offset && - entry.offset <= selectionOffsetEnd - tempEntries = { - ...tempEntries, - [key]: { - ...tempEntries[key], - focused: entry.offset === selectionOffsetStart, - }, - } - } else if ( - ['add-comment', 'bulk-actions'].includes(entry.type) && - selection - ) { - tempEntries = { - ...tempEntries, - [key]: { ...tempEntries[key], focused: true }, - } - } - if (isChangeEntryAndWithinSelection) { - const entryIds = 'entry_ids' in entry ? entry.entry_ids : [] - for (const entryId of entryIds) { - selectedEntryIds.current.push(entryId) - } - tempNVisibleSelectedChanges++ - } - } - - // `tempEntries` would be mutated - dispatchReviewPanelEvent('recalculate-screen-positions', { - entries: tempEntries, - updateType, - }) - - // The state should be updated after dispatching the 'recalculate-screen-positions' - // event as `tempEntries` will be mutated - setEntries(prev => ({ ...prev, [currentDocumentId]: tempEntries })) - setNVisibleSelectedChanges(tempNVisibleSelectedChanges) - - handleLayoutChange() - } - - const addNewCommentFromKbdShortcut = () => { - if (!trackChangesVisible) { - return - } - dispatchReviewPanelEvent('comment:select_line') - - if (!reviewPanelOpen) { - toggleReviewPanel() - } - handleLayoutChange({ async: true }) - addCommentEmitter() - } - - const handleEditorEvents = (e: Event) => { - const event = e as CustomEvent - const { type, payload } = event.detail - - switch (type) { - case 'line-height': { - editorLineHeightChanged(payload) - break - } - - case 'track-changes:changed': { - editorTrackChangesChanged() - break - } - - case 'track-changes:visibility_changed': { - editorTrackChangesVisibilityChanged() - break - } - - case 'focus:changed': { - const { from, to, empty, updateType } = payload - editorFocusChanged(from, to, !empty, updateType) - break - } - - case 'add-new-comment': { - addNewCommentFromKbdShortcut() - break - } - - case 'toggle-track-changes': { - toggleTrackChangesFromKbdShortcut() - break - } - - case 'toggle-review-panel': { - toggleReviewPanel() - break - } - } - } - - window.addEventListener('editor:event', handleEditorEvents) - - return () => { - window.removeEventListener('editor:event', handleEditorEvents) - } - }, [ - addCommentEmitter, - currentDocumentId, - getDocEntries, - resolvedThreadIds, - reviewPanelOpen, - toggleReviewPanel, - toggleTrackChangesForUser, - trackChanges, - trackChangesState, - trackChangesVisible, - updateEntries, - user.id, - ]) - - useSocketListener(socket, 'reopen-thread', onCommentReopened) - useSocketListener(socket, 'delete-thread', onThreadDeleted) - useSocketListener(socket, 'resolve-thread', onCommentResolved) - useSocketListener(socket, 'edit-message', onCommentEdited) - useSocketListener(socket, 'delete-message', onCommentDeleted) - useSocketListener( - socket, - 'accept-changes', - useCallback( - (docId: DocId, entryIds: ThreadId[]) => { - if (docId !== currentDocumentId) { - getChangeTracker(docId).removeChangeIds(entryIds) - } else { - dispatchReviewPanelEvent('changes:accept', entryIds) - } - updateEntries(docId) - }, - [currentDocumentId, getChangeTracker, updateEntries] - ) - ) - useSocketListener( - socket, - 'new-comment', - useCallback( - (threadId: ThreadId, comment: ReviewPanelCommentThreadMessageApi) => { - setCommentThreads(prevState => { - const { submitting: _, ...thread } = getThread(threadId) - thread.messages = [...thread.messages] - thread.messages.push(formatComment(comment)) - return { ...prevState, [threadId]: thread } - }) - handleLayoutChange({ async: true }) - }, - [getThread] - ) - ) - useSocketListener( - socket, - 'new-comment-threads', - useCallback( - (threads: ReviewPanelCommentThreadsApi) => { - setCommentThreads(prevState => { - const newThreads = { ...prevState } - for (const threadIdString of Object.keys(threads)) { - const threadId = threadIdString as ThreadId - const { submitting: _, ...thread } = getThread(threadId) - // Replace already loaded messages with the server provided ones - thread.messages = threads[threadId].messages.map(formatComment) - newThreads[threadId] = thread - } - return newThreads - }) - handleLayoutChange({ async: true }) - }, - [getThread] - ) - ) - - const openSubView = useRef('cur_file') - useEffect(() => { - if (!reviewPanelOpen) { - // Always show current file when not open, but save current state - setSubView(prevState => { - openSubView.current = prevState - return 'cur_file' - }) - } else { - // Reset back to what we had when previously open - setSubView(openSubView.current) - } - handleLayoutChange({ async: true, animate: false }) - }, [reviewPanelOpen]) - - const canRefreshRanges = useRef(false) - const prevSubView = useRef(subView) - const initializedPrevSubView = useRef(false) - useEffect(() => { - // Prevent setting a computed value for `prevSubView` on mount - if (!initializedPrevSubView.current) { - initializedPrevSubView.current = true - return - } - prevSubView.current = subView === 'cur_file' ? 'overview' : 'cur_file' - // Allow refreshing ranges once for each `subView` change - canRefreshRanges.current = true - }, [subView]) - - useEffect(() => { - if (subView === 'overview' && canRefreshRanges.current) { - canRefreshRanges.current = false - - setIsOverviewLoading(true) - refreshRanges().finally(() => { - setIsOverviewLoading(false) - }) - } - }, [subView, refreshRanges]) - - useEffect(() => { - if (subView === 'cur_file' && prevSubView.current === 'overview') { - dispatchReviewPanelEvent('overview-closed', subView) - } - }, [subView]) - - useEffect(() => { - if (Object.keys(users).length) { - handleLayoutChange({ async: true }) - } - }, [users]) - - const values = useMemo( - () => ({ - collapsed, - commentThreads, - entries, - isAddingComment, - loadingThreads, - nVisibleSelectedChanges, - permissions, - users, - resolvedComments, - shouldCollapse, - navHeight, - toolbarHeight, - subView, - wantTrackChanges, - isOverviewLoading, - openDocId: currentDocumentId, - lineHeight, - trackChangesState, - trackChangesOnForEveryone, - trackChangesOnForGuests, - trackChangesForGuestsAvailable, - formattedProjectMembers, - layoutSuspended, - unsavedComment, - layoutToLeft, - }), - [ - collapsed, - commentThreads, - entries, - isAddingComment, - loadingThreads, - nVisibleSelectedChanges, - permissions, - users, - resolvedComments, - shouldCollapse, - navHeight, - toolbarHeight, - subView, - wantTrackChanges, - isOverviewLoading, - currentDocumentId, - lineHeight, - trackChangesState, - trackChangesOnForEveryone, - trackChangesOnForGuests, - trackChangesForGuestsAvailable, - formattedProjectMembers, - layoutSuspended, - unsavedComment, - layoutToLeft, - ] - ) - - const updaterFns = useMemo( - () => ({ - handleSetSubview, - handleLayoutChange, - gotoEntry, - resolveComment, - submitReply, - acceptChanges, - rejectChanges, - toggleReviewPanel, - bulkAcceptActions, - bulkRejectActions, - saveEdit, - submitNewComment, - deleteComment, - unresolveComment, - refreshResolvedCommentsDropdown: refreshRanges, - deleteThread, - toggleTrackChangesForEveryone, - toggleTrackChangesForUser, - toggleTrackChangesForGuests, - setCollapsed, - setShouldCollapse, - setIsAddingComment, - setNavHeight, - setToolbarHeight, - setLayoutSuspended, - setUnsavedComment, - }), - [ - handleSetSubview, - gotoEntry, - resolveComment, - submitReply, - acceptChanges, - rejectChanges, - toggleReviewPanel, - bulkAcceptActions, - bulkRejectActions, - saveEdit, - submitNewComment, - deleteComment, - unresolveComment, - refreshRanges, - deleteThread, - toggleTrackChangesForEveryone, - toggleTrackChangesForUser, - toggleTrackChangesForGuests, - setCollapsed, - setShouldCollapse, - setIsAddingComment, - setNavHeight, - setToolbarHeight, - setLayoutSuspended, - setUnsavedComment, - ] - ) - - return { values, updaterFns } -} - -export default useReviewPanelState diff --git a/services/web/modules/track-changes/app/src/TrackChangesController.js b/services/web/modules/track-changes/app/src/TrackChangesController.js index 12cbb57da49..45f8a03e0f0 100644 --- a/services/web/modules/track-changes/app/src/TrackChangesController.js +++ b/services/web/modules/track-changes/app/src/TrackChangesController.js @@ -3,6 +3,7 @@ const ChatManager = require('../../../../app/src/Features/Chat/ChatManager') const EditorRealTimeController = require('../../../../app/src/Features/Editor/EditorRealTimeController') const SessionManager = require('../../../../app/src/Features/Authentication/SessionManager') const UserInfoManager = require('../../../../app/src/Features/User/UserInfoManager') +const UserInfoController = require('../../../../app/src/Features/User/UserInfoController') const DocstoreManager = require('../../../../app/src/Features/Docstore/DocstoreManager') const DocumentUpdaterHandler = require('../../../../app/src/Features/DocumentUpdater/DocumentUpdaterHandler') const CollaboratorsGetter = require('../../../../app/src/Features/Collaborators/CollaboratorsGetter') @@ -11,10 +12,10 @@ const pLimit = require('p-limit') function _transformId(doc) { if (doc._id) { - doc.id = doc._id; - delete doc._id; + doc.id = doc._id + delete doc._id } - return doc; + return doc } const TrackChangesController = { @@ -44,7 +45,6 @@ const TrackChangesController = { async getAllRanges(req, res, next) { try { const { project_id } = req.params - // Flushing the project to mongo is not ideal. Is it possible to fetch the ranges from redis? await DocumentUpdaterHandler.promises.flushProjectToMongo(project_id) const ranges = await DocstoreManager.promises.getAllRanges(project_id) res.json(ranges.map(_transformId)) @@ -53,31 +53,12 @@ const TrackChangesController = { } }, async getChangesUsers(req, res, next) { +// This route was previously used by the frontend to retrieve names of users who made changes or comments. +// review-panel-new no longer needs this for comments, but still relies on it for changes - +// although the frontend knows the names of the current owner and members, it depends on the data +// provided here to assign names to authors who have left the project but still have unaccepted changes. try { const { project_id } = req.params - const memberIds = await CollaboratorsGetter.promises.getMemberIds(project_id) - // FIXME: Fails to display names in changes made by former project collaborators. - // See the alternative below. However, it requires flushing the project to mongo, which is not ideal. - const limit = pLimit(3) - const users = await Promise.all( - memberIds.map(memberId => - limit(async () => { - const user = await UserInfoManager.promises.getPersonalInfo(memberId) - return user - }) - ) - ) - users.push({_id: null}) // An anonymous user won't cause any harm - res.json(users.map(_transformId)) - } catch (err) { - next(err) - } - }, -/* - async getChangesUsers(req, res, next) { - try { - const { project_id } = req.params - await DocumentUpdaterHandler.promises.flushProjectToMongo(project_id) const memberIds = new Set() const ranges = await DocstoreManager.promises.getAllRanges(project_id) ranges.forEach(range => { @@ -89,20 +70,16 @@ const TrackChangesController = { const users = await Promise.all( [...memberIds].map(memberId => limit(async () => { - if( memberId !== "anonymous-user") { - return await UserInfoManager.promises.getPersonalInfo(memberId) - } else { - return {_id: null} - } + const user = await UserInfoManager.promises.getPersonalInfo(memberId) + return UserInfoController.formatPersonalInfo(user) }) ) ) - res.json(users.map(_transformId)) + res.json(users) } catch (err) { next(err) } }, -*/ async getThreads(req, res, next) { try { const { project_id } = req.params @@ -120,11 +97,12 @@ const TrackChangesController = { const user_id = SessionManager.getLoggedInUserId(req.session) if (!user_id) throw new Error('no logged-in user') const message = await ChatApiHandler.promises.sendComment(project_id, thread_id, user_id, content) - message.user = await UserInfoManager.promises.getPersonalInfo(user_id) + const user = await UserInfoManager.promises.getPersonalInfo(user_id) + message.user = UserInfoController.formatPersonalInfo(user) EditorRealTimeController.emitToRoom(project_id, 'new-comment', thread_id, message) res.sendStatus(204) } catch (err) { - next(err); + next(err) } }, async editMessage(req, res, next) { @@ -157,11 +135,16 @@ const TrackChangesController = { if (!user_id) throw new Error('no logged-in user') const user = await UserInfoManager.promises.getPersonalInfo(user_id) await ChatApiHandler.promises.resolveThread(project_id, thread_id, user_id) - EditorRealTimeController.emitToRoom(project_id, 'resolve-thread', thread_id, user) + EditorRealTimeController.emitToRoom( + project_id, + 'resolve-thread', + thread_id, + UserInfoController.formatPersonalInfo(user) + ) await DocumentUpdaterHandler.promises.resolveThread(project_id, doc_id, thread_id, user_id) - res.sendStatus(204); + res.sendStatus(204) } catch (err) { - next(err); + next(err) } }, async reopenThread(req, res, next) { From 93734a66c2de7e2192e8ce3add5ee91f0c711350 Mon Sep 17 00:00:00 2001 From: yu-i-i Date: Fri, 2 May 2025 02:05:34 +0200 Subject: [PATCH 019/106] Add Template Gallery support --- develop/docker-compose.yml | 2 +- services/filestore/app.js | 5 + services/filestore/app/js/FileConverter.js | 2 +- services/filestore/app/js/FileHandler.js | 4 +- .../Features/Project/ProjectController.mjs | 7 +- .../Templates/TemplatesController.mjs | 29 ++- .../Features/Templates/TemplatesManager.mjs | 87 +++---- .../app/src/infrastructure/ExpressLocals.mjs | 2 +- .../web/app/src/infrastructure/Features.mjs | 2 +- services/web/app/src/router.mjs | 2 + .../web/app/views/layout/navbar-marketing.pug | 11 + .../project/editor/new_from_template.pug | 4 +- .../template_gallery/template-gallery.pug | 18 ++ .../app/views/template_gallery/template.pug | 20 ++ services/web/config/settings.defaults.js | 3 +- .../web/frontend/extracted-translations.json | 18 ++ .../components/actions-manage-template.tsx | 72 ++++++ .../editor-manage-template-modal-wrapper.jsx | 37 +++ .../manage-template-modal-content.jsx | 203 ++++++++++++++++ .../components/manage-template-modal.jsx | 51 ++++ .../components/settings-template-category.tsx | 51 ++++ .../components/gallery-header-all.tsx | 29 +++ .../components/gallery-header-tagged.tsx | 33 +++ .../components/gallery-popular-tags.tsx | 29 +++ .../components/gallery-search-sort-header.tsx | 78 ++++++ .../components/pagination.tsx | 80 +++++++ .../components/search-form.tsx | 59 +++++ .../components/sort/with-content.tsx | 46 ++++ .../components/template-gallery-entry.tsx | 29 +++ .../components/template-gallery-root.tsx | 64 +++++ .../components/template-gallery.tsx | 65 +++++ .../context/template-gallery-context.tsx | 129 ++++++++++ .../template-gallery/hooks/use-sort.ts | 20 ++ .../js/features/template-gallery/types/api.ts | 12 + .../js/features/template-gallery/util/api.ts | 12 + .../template-gallery/util/sort-templates.ts | 40 ++++ .../components/delete-template-button.tsx | 48 ++++ .../components/delete-template-modal.tsx | 43 ++++ .../components/edit-template-button.tsx | 46 ++++ .../components/edit-template-modal.tsx | 153 ++++++++++++ .../components/settings/settings-language.tsx | 33 +++ .../settings/settings-menu-select.tsx | 106 ++++++++ .../settings/settings-template-category.tsx | 44 ++++ .../components/template-action-modal.tsx | 136 +++++++++++ .../template/components/template-details.tsx | 103 ++++++++ .../template/components/template-preview.tsx | 23 ++ .../template/components/template-root.tsx | 70 ++++++ .../template/context/template-context.tsx | 53 ++++ .../frontend/js/features/template/util/api.ts | 47 ++++ .../frontend/js/pages/template-gallery.tsx | 14 ++ services/web/frontend/js/pages/template.tsx | 14 ++ .../components/navbar/logged-in-items.tsx | 3 + .../components/navbar/logged-out-items.tsx | 3 + services/web/frontend/js/utils/meta.ts | 2 + .../frontend/stylesheets/components/link.scss | 1 + .../stylesheets/pages/templates-v2.scss | 26 ++ services/web/locales/de.json | 14 ++ services/web/locales/en.json | 14 +- services/web/locales/ru.json | 16 +- .../template-gallery/app/src/CleanHtml.mjs | 22 ++ .../app/src/TemplateErrors.mjs | 13 + .../app/src/TemplateGalleryController.mjs | 162 +++++++++++++ .../app/src/TemplateGalleryHelper.mjs | 226 ++++++++++++++++++ .../app/src/TemplateGalleryManager.mjs | 214 +++++++++++++++++ .../app/src/TemplateGalleryRouter.mjs | 77 ++++++ .../app/src/models/Template.js | 33 +++ .../web/modules/template-gallery/index.mjs | 33 +++ services/web/types/template.ts | 16 ++ 68 files changed, 3098 insertions(+), 65 deletions(-) create mode 100644 services/web/app/views/template_gallery/template-gallery.pug create mode 100644 services/web/app/views/template_gallery/template.pug create mode 100644 services/web/frontend/js/features/editor-left-menu/components/actions-manage-template.tsx create mode 100644 services/web/frontend/js/features/manage-template-modal/components/editor-manage-template-modal-wrapper.jsx create mode 100644 services/web/frontend/js/features/manage-template-modal/components/manage-template-modal-content.jsx create mode 100644 services/web/frontend/js/features/manage-template-modal/components/manage-template-modal.jsx create mode 100644 services/web/frontend/js/features/manage-template-modal/components/settings-template-category.tsx create mode 100644 services/web/frontend/js/features/template-gallery/components/gallery-header-all.tsx create mode 100644 services/web/frontend/js/features/template-gallery/components/gallery-header-tagged.tsx create mode 100644 services/web/frontend/js/features/template-gallery/components/gallery-popular-tags.tsx create mode 100644 services/web/frontend/js/features/template-gallery/components/gallery-search-sort-header.tsx create mode 100644 services/web/frontend/js/features/template-gallery/components/pagination.tsx create mode 100644 services/web/frontend/js/features/template-gallery/components/search-form.tsx create mode 100644 services/web/frontend/js/features/template-gallery/components/sort/with-content.tsx create mode 100644 services/web/frontend/js/features/template-gallery/components/template-gallery-entry.tsx create mode 100644 services/web/frontend/js/features/template-gallery/components/template-gallery-root.tsx create mode 100644 services/web/frontend/js/features/template-gallery/components/template-gallery.tsx create mode 100644 services/web/frontend/js/features/template-gallery/context/template-gallery-context.tsx create mode 100644 services/web/frontend/js/features/template-gallery/hooks/use-sort.ts create mode 100644 services/web/frontend/js/features/template-gallery/types/api.ts create mode 100644 services/web/frontend/js/features/template-gallery/util/api.ts create mode 100644 services/web/frontend/js/features/template-gallery/util/sort-templates.ts create mode 100644 services/web/frontend/js/features/template/components/delete-template-button.tsx create mode 100644 services/web/frontend/js/features/template/components/delete-template-modal.tsx create mode 100644 services/web/frontend/js/features/template/components/edit-template-button.tsx create mode 100644 services/web/frontend/js/features/template/components/edit-template-modal.tsx create mode 100644 services/web/frontend/js/features/template/components/settings/settings-language.tsx create mode 100644 services/web/frontend/js/features/template/components/settings/settings-menu-select.tsx create mode 100644 services/web/frontend/js/features/template/components/settings/settings-template-category.tsx create mode 100644 services/web/frontend/js/features/template/components/template-action-modal.tsx create mode 100644 services/web/frontend/js/features/template/components/template-details.tsx create mode 100644 services/web/frontend/js/features/template/components/template-preview.tsx create mode 100644 services/web/frontend/js/features/template/components/template-root.tsx create mode 100644 services/web/frontend/js/features/template/context/template-context.tsx create mode 100644 services/web/frontend/js/features/template/util/api.ts create mode 100644 services/web/frontend/js/pages/template-gallery.tsx create mode 100644 services/web/frontend/js/pages/template.tsx create mode 100644 services/web/modules/template-gallery/app/src/CleanHtml.mjs create mode 100644 services/web/modules/template-gallery/app/src/TemplateErrors.mjs create mode 100644 services/web/modules/template-gallery/app/src/TemplateGalleryController.mjs create mode 100644 services/web/modules/template-gallery/app/src/TemplateGalleryHelper.mjs create mode 100644 services/web/modules/template-gallery/app/src/TemplateGalleryManager.mjs create mode 100644 services/web/modules/template-gallery/app/src/TemplateGalleryRouter.mjs create mode 100644 services/web/modules/template-gallery/app/src/models/Template.js create mode 100644 services/web/modules/template-gallery/index.mjs create mode 100644 services/web/types/template.ts diff --git a/develop/docker-compose.yml b/develop/docker-compose.yml index 3c8c0787192..96d0bf9b2dd 100644 --- a/develop/docker-compose.yml +++ b/develop/docker-compose.yml @@ -131,7 +131,7 @@ services: dockerfile: services/real-time/Dockerfile env_file: - dev.env - + redis: image: redis:7 ports: diff --git a/services/filestore/app.js b/services/filestore/app.js index 9d138c81113..bdcd2930c4a 100644 --- a/services/filestore/app.js +++ b/services/filestore/app.js @@ -66,6 +66,11 @@ if (settings.filestore.stores.template_files) { keyBuilder.templateFileKeyMiddleware, fileController.insertFile ) + app.delete( + '/template/:template_id/v/:version/:format', + keyBuilder.templateFileKeyMiddleware, + fileController.deleteFile + ) } app.get( diff --git a/services/filestore/app/js/FileConverter.js b/services/filestore/app/js/FileConverter.js index cdde44b1e75..6030d378c57 100644 --- a/services/filestore/app/js/FileConverter.js +++ b/services/filestore/app/js/FileConverter.js @@ -6,7 +6,7 @@ import Errors from './Errors.js' const { ConversionError } = Errors -const APPROVED_FORMATS = ['png'] +const APPROVED_FORMATS = ['png', 'jpg'] const FOURTY_SECONDS = 40 * 1000 const KILL_SIGNAL = 'SIGTERM' diff --git a/services/filestore/app/js/FileHandler.js b/services/filestore/app/js/FileHandler.js index 5e70afb3ad6..44c8f03caf2 100644 --- a/services/filestore/app/js/FileHandler.js +++ b/services/filestore/app/js/FileHandler.js @@ -111,7 +111,9 @@ async function _getConvertedFileAndCache(bucket, key, convertedKey, opts) { let convertedFsPath try { convertedFsPath = await _convertFile(bucket, key, opts) - await ImageOptimiser.promises.compressPng(convertedFsPath) + if (convertedFsPath.toLowerCase().endsWith(".png")) { + await ImageOptimiser.promises.compressPng(convertedFsPath) + } await PersistorManager.sendFile(bucket, convertedKey, convertedFsPath) } catch (err) { LocalFileWriter.deleteFile(convertedFsPath, () => {}) diff --git a/services/web/app/src/Features/Project/ProjectController.mjs b/services/web/app/src/Features/Project/ProjectController.mjs index 32e642f132d..42bb25497c8 100644 --- a/services/web/app/src/Features/Project/ProjectController.mjs +++ b/services/web/app/src/Features/Project/ProjectController.mjs @@ -741,11 +741,16 @@ const _ProjectController = { ) } +console.log("Features.hasFeature('templates-server-pro') = ", Features.hasFeature('templates-server-pro')) +console.log("Settings.templates?.nonAdminCanManage", Settings.templates?.nonAdminCanManage) + const isAdminOrTemplateOwner = - hasAdminAccess(user) || Settings.templates?.user_id === userId + hasAdminAccess(user) || Settings.templates?.nonAdminCanManage const showTemplatesServerPro = Features.hasFeature('templates-server-pro') && isAdminOrTemplateOwner +console.log("showTemplatesServerPro = ", showTemplatesServerPro ) + const debugPdfDetach = shouldDisplayFeature('debug_pdf_detach') const detachRole = req.params.detachRole diff --git a/services/web/app/src/Features/Templates/TemplatesController.mjs b/services/web/app/src/Features/Templates/TemplatesController.mjs index 85fb1fb4b34..2c504cd132b 100644 --- a/services/web/app/src/Features/Templates/TemplatesController.mjs +++ b/services/web/app/src/Features/Templates/TemplatesController.mjs @@ -7,21 +7,22 @@ import { expressify } from '@overleaf/promise-utils' const TemplatesController = { async getV1Template(req, res) { - const templateVersionId = req.params.Template_version_id - const templateId = req.query.id - if (!/^[0-9]+$/.test(templateVersionId) || !/^[0-9]+$/.test(templateId)) { - logger.err( - { templateVersionId, templateId }, - 'invalid template id or version' - ) - return res.sendStatus(400) - } + const templateId = req.params.Template_version_id + const templateVersionId = req.query.version +// if (!/^[0-9]+$/.test(templateVersionId) || !/^[0-9]+$/.test(templateId)) { +// logger.err( +// { templateVersionId, templateId }, +// 'invalid template id or version' +// ) +// return res.sendStatus(400) +// } const data = { templateVersionId, templateId, - name: req.query.templateName, - compiler: ProjectHelper.compilerFromV1Engine(req.query.latexEngine), - imageName: req.query.texImage, + name: req.query.name, + compiler: req.query.compiler, + language: req.query.language, + imageName: req.query.imageName, mainFile: req.query.mainFile, brandVariationId: req.query.brandVariationId, } @@ -36,6 +37,7 @@ const TemplatesController = { async createProjectFromV1Template(req, res) { const userId = SessionManager.getLoggedInUserId(req.session) + const project = await TemplatesManager.promises.createProjectFromV1Template( req.body.brandVariationId, req.body.compiler, @@ -44,7 +46,8 @@ const TemplatesController = { req.body.templateName, req.body.templateVersionId, userId, - req.body.imageName + req.body.imageName, + req.body.language ) delete req.session.templateData if (!project) { diff --git a/services/web/app/src/Features/Templates/TemplatesManager.mjs b/services/web/app/src/Features/Templates/TemplatesManager.mjs index 0781597cc5a..ce6a5a32f59 100644 --- a/services/web/app/src/Features/Templates/TemplatesManager.mjs +++ b/services/web/app/src/Features/Templates/TemplatesManager.mjs @@ -20,6 +20,7 @@ import OError from '@overleaf/o-error' const { promises: ProjectRootDocManager } = ProjectRootDocManagerModule const { promises: ProjectOptionsHandler } = ProjectOptionsHandlerModule +const TIMEOUT = 30000 // 30 sec const TemplatesManager = { async createProjectFromV1Template( @@ -30,34 +31,19 @@ const TemplatesManager = { templateName, templateVersionId, userId, - imageName + imageName, + language ) { - compiler = ProjectOptionsHandler.normalizeCompiler(compiler || 'pdflatex') - imageName = ProjectOptionsHandler.normalizeImageName( - imageName || 'wl_texlive:2018.1' - ) - - const zipUrl = `${settings.apis.v1.url}/api/v1/overleaf/templates/${templateVersionId}` + const zipUrl = `${settings.apis.filestore.url}/template/${templateId}/v/${templateVersionId}/zip` const zipReq = await fetchStreamWithResponse(zipUrl, { - basicAuth: { - user: settings.apis.v1.user, - password: settings.apis.v1.pass, - }, - signal: AbortSignal.timeout(settings.apis.v1.timeout), + signal: AbortSignal.timeout(TIMEOUT), }) const projectName = ProjectDetailsHandler.fixProjectName(templateName) const dumpPath = `${settings.path.dumpFolder}/${crypto.randomUUID()}` const writeStream = fs.createWriteStream(dumpPath) try { - const attributes = { - fromV1TemplateId: templateId, - fromV1TemplateVersionId: templateVersionId, - compiler, - imageName, - } - if (brandVariationId) attributes.brandVariationId = brandVariationId - + const attributes = {} await pipeline(zipReq.stream, writeStream) if (zipReq.response.status !== 200) { @@ -87,22 +73,13 @@ const TemplatesManager = { return undefined }) - await TemplatesManager._setMainFile(project, mainFile) - - const found = await prepareClsiCacheInBackground - if (found === false && project.rootDoc_id) { - ClsiCacheManager.createTemplateClsiCache({ - templateVersionId, - project, - fileEntries, - docEntries, - }).catch(err => { - logger.error( - { err, templateVersionId }, - 'failed to create template clsi-cache' - ) - }) - } + await TemplatesManager._setCompiler(project._id, compiler) + await TemplatesManager._setImage(project._id, imageName) + await TemplatesManager._setMainFile(project._id, mainFile) + await TemplatesManager._setSpellCheckLanguage(project._id, language) + await TemplatesManager._setBrandVariationId(project._id, brandVariationId) + + await prepareClsiCacheInBackground return project } finally { @@ -110,15 +87,41 @@ const TemplatesManager = { } }, - async _setMainFile(project, mainFile) { + async _setCompiler(projectId, compiler) { + if (compiler == null) { + return + } + await ProjectOptionsHandler.setCompiler(projectId, compiler) + }, + + async _setImage(projectId, imageName) { + try { + await ProjectOptionsHandler.setImageName(projectId, imageName) + } catch { + logger.warn({ imageName: imageName }, 'not available') + await ProjectOptionsHandler.setImageName(projectId, process.env.TEX_LIVE_DOCKER_IMAGE) + } + }, + + async _setMainFile(projectId, mainFile) { if (mainFile == null) { return } - const rootDocId = await ProjectRootDocManager.setRootDocFromName( - project._id, - mainFile - ) - if (rootDocId) project.rootDoc_id = rootDocId + await ProjectRootDocManager.setRootDocFromName(projectId, mainFile) + }, + + async _setSpellCheckLanguage(projectId, language) { + if (language == null) { + return + } + await ProjectOptionsHandler.setSpellCheckLanguage(projectId, language) + }, + + async _setBrandVariationId(projectId, brandVariationId) { + if (brandVariationId == null) { + return + } + await ProjectOptionsHandler.setBrandVariationId(projectId, brandVariationId) }, async fetchFromV1(templateId) { diff --git a/services/web/app/src/infrastructure/ExpressLocals.mjs b/services/web/app/src/infrastructure/ExpressLocals.mjs index 2907f4ac08b..41583a48131 100644 --- a/services/web/app/src/infrastructure/ExpressLocals.mjs +++ b/services/web/app/src/infrastructure/ExpressLocals.mjs @@ -413,7 +413,7 @@ export default async function (webRouter, privateApiRouter, publicApiRouter) { labsEnabled: Settings.labs && Settings.labs.enable, wikiEnabled: Settings.overleaf != null || Settings.proxyLearn, templatesEnabled: - Settings.overleaf != null || Settings.templates?.user_id != null, + Settings.overleaf != null || Boolean(Settings.moduleImportSequence.includes('template-gallery')), cioWriteKey: Settings.analytics?.cio?.writeKey, cioSiteId: Settings.analytics?.cio?.siteId, linkedInInsightsPartnerId: Settings.analytics?.linkedIn?.partnerId, diff --git a/services/web/app/src/infrastructure/Features.mjs b/services/web/app/src/infrastructure/Features.mjs index 229312f38c7..193cc3c658d 100644 --- a/services/web/app/src/infrastructure/Features.mjs +++ b/services/web/app/src/infrastructure/Features.mjs @@ -68,7 +68,7 @@ const Features = { case 'oauth': return Boolean(Settings.oauth) case 'templates-server-pro': - return Boolean(Settings.templates?.user_id) + return Boolean(Settings.moduleImportSequence.includes('template-gallery')) case 'affiliations': case 'analytics': return Boolean(_.get(Settings, ['apis', 'v1', 'url'])) diff --git a/services/web/app/src/router.mjs b/services/web/app/src/router.mjs index 41f74d79bfc..721a62aba11 100644 --- a/services/web/app/src/router.mjs +++ b/services/web/app/src/router.mjs @@ -275,6 +275,8 @@ async function initialize(webRouter, privateApiRouter, publicApiRouter) { '/read-only/one-time-login' ) + await Modules.applyRouter(webRouter, privateApiRouter, publicApiRouter) + webRouter.get('/logout', UserPagesController.logout) webRouter.post('/logout', UserController.logout) diff --git a/services/web/app/views/layout/navbar-marketing.pug b/services/web/app/views/layout/navbar-marketing.pug index d99e4db5802..84f037402a6 100644 --- a/services/web/app/views/layout/navbar-marketing.pug +++ b/services/web/app/views/layout/navbar-marketing.pug @@ -149,6 +149,17 @@ nav.navbar.navbar-default.navbar-main.navbar-expand-lg( // logged out if !getSessionUser() + // templates link + li + a( + href="/templates" + event-tracking="menu-click" + event-tracking-action="clicked" + event-tracking-trigger="click" + event-tracking-mb="true" + event-segmentation={ page: currentUrl, item: 'templates', location: 'top-menu' } + ) #{translate('templates')} + // register link if hasFeature('registration-page') +nav-item.primary diff --git a/services/web/app/views/project/editor/new_from_template.pug b/services/web/app/views/project/editor/new_from_template.pug index a5dc3ff33c3..6d359131412 100644 --- a/services/web/app/views/project/editor/new_from_template.pug +++ b/services/web/app/views/project/editor/new_from_template.pug @@ -29,8 +29,10 @@ block content input(type="hidden" name="templateVersionId" value=templateVersionId) input(type="hidden" name="templateName" value=name) input(type="hidden" name="compiler" value=compiler) - input(type="hidden" name="imageName" value=imageName) + if imageName + input(type="hidden" name="imageName" value=imageName) input(type="hidden" name="mainFile" value=mainFile) + input(type="hidden" name="language" value=language) if brandVariationId input(type="hidden" name="brandVariationId" value=brandVariationId) input(hidden type="submit") diff --git a/services/web/app/views/template_gallery/template-gallery.pug b/services/web/app/views/template_gallery/template-gallery.pug new file mode 100644 index 00000000000..3838d30606d --- /dev/null +++ b/services/web/app/views/template_gallery/template-gallery.pug @@ -0,0 +1,18 @@ +extends ../layout-react + +block entrypointVar + - entrypoint = 'pages/template-gallery' + +block vars +block vars + - const suppressNavContentLinks = true + - const suppressNavbar = true + - const suppressFooter = true + - bootstrap5PageStatus = 'enabled' // One of 'disabled', 'enabled', and 'queryStringOnly' + - isWebsiteRedesign = false + +block append meta + meta(name="ol-templateCategory" data-type="string" content=category) + +block content + #template-gallery-root diff --git a/services/web/app/views/template_gallery/template.pug b/services/web/app/views/template_gallery/template.pug new file mode 100644 index 00000000000..e56fd8d2e57 --- /dev/null +++ b/services/web/app/views/template_gallery/template.pug @@ -0,0 +1,20 @@ +extends ../layout-react + +block entrypointVar + - entrypoint = 'pages/template' + +block vars + - const suppressNavContentLinks = true + - const suppressNavbar = true + - const suppressFooter = true + - bootstrap5PageStatus = 'enabled' // One of 'disabled', 'enabled', and 'queryStringOnly' + - isWebsiteRedesign = false + +block append meta + meta(name="ol-template" data-type="json" content=template) + meta(name="ol-languages" data-type="json" content=languages) + meta(name="ol-userIsAdmin" data-type="boolean" content=hasAdminAccess()) + +block content + #template-root + diff --git a/services/web/config/settings.defaults.js b/services/web/config/settings.defaults.js index ce217b79fa0..6b69d11fbf5 100644 --- a/services/web/config/settings.defaults.js +++ b/services/web/config/settings.defaults.js @@ -1018,7 +1018,7 @@ module.exports = { importProjectFromGithubModalWrapper: [], importProjectFromGithubMenu: [], editorLeftMenuSync: [], - editorLeftMenuManageTemplate: [], + editorLeftMenuManageTemplate: ['@/features/editor-left-menu/components/actions-manage-template'], menubarExtraComponents: [], oauth2Server: [], managedGroupSubscriptionEnrollmentNotification: [], @@ -1075,6 +1075,7 @@ module.exports = { 'authentication/ldap', 'authentication/saml', 'authentication/oidc', + 'template-gallery', ], viewIncludes: {}, diff --git a/services/web/frontend/extracted-translations.json b/services/web/frontend/extracted-translations.json index 9380abf2e57..4af2b2e0583 100644 --- a/services/web/frontend/extracted-translations.json +++ b/services/web/frontend/extracted-translations.json @@ -26,6 +26,7 @@ "about_to_delete_cert": "", "about_to_delete_projects": "", "about_to_delete_tag": "", + "about_to_delete_template": "", "about_to_delete_the_following_project": "", "about_to_delete_the_following_projects": "", "about_to_delete_user_preamble": "", @@ -140,6 +141,7 @@ "all_project_activity_description": "", "all_projects": "", "all_projects_will_be_transferred_immediately": "", + "all_templates": "", "all_these_experiments_are_available_exclusively": "", "allocate_license": "", "allows_to_search_by_author_title_etc_possible_to_pull_results_directly_from_your_reference_manager_if_connected": "", @@ -176,6 +178,7 @@ "at_most_x_libraries_can_be_selected": "", "attach_image_or_pdf": "", "audit_logs": "", + "author": "", "auto_close_brackets": "", "auto_compile": "", "auto_complete": "", @@ -249,6 +252,7 @@ "card_must_be_authenticated_by_3dsecure": "", "card_payment": "", "careers": "", + "categories": "", "category_arrows": "", "category_greek": "", "category_misc": "", @@ -401,6 +405,7 @@ "cut": "", "dark_mode_pdf_preview": "", "dark_themes": "", + "date": "", "date_and_owner": "", "date_and_time": "", "dealing_with_errors": "", @@ -430,6 +435,7 @@ "delete_sso_config": "", "delete_table": "", "delete_tag": "", + "delete_template": "", "delete_token": "", "delete_user": "", "delete_your_account": "", @@ -535,6 +541,7 @@ "edit_sso_configuration": "", "edit_tag": "", "edit_tag_name": "", + "edit_template": "", "edit_your_custom_dictionary": "", "edited": "", "editing": "", @@ -987,6 +994,7 @@ "last_name": "", "last_resort_trouble_shooting_guide": "", "last_suggested_fix": "", + "last_updated": "", "last_updated_date_by_x": "", "last_used": "", "latam_discount_modal_info": "", @@ -994,6 +1002,8 @@ "latex_in_thirty_minutes": "", "latex_places_figures_according_to_a_special_algorithm": "", "latex_places_tables_according_to_a_special_algorithm": "", + "latex_templates": "", + "latex_templates_for_journal_articles": "", "layout_options": "", "learn_more": "", "learn_more_about": "", @@ -1214,6 +1224,7 @@ "no_selection_select_file": "", "no_symbols_found": "", "no_thanks_cancel_now": "", + "no_templates_found": "", "normal": "", "normally_x_price_per_month": "", "normally_x_price_per_year": "", @@ -1245,6 +1256,7 @@ "only_importer_can_refresh": "", "open_action_menu": "", "open_advanced_reference_search": "", + "open_as_template": "", "open_file": "", "open_link": "", "open_path": "", @@ -1269,6 +1281,7 @@ "overleaf_is_easy_to_use": "", "overleaf_labs": "", "overleaf_logo": "", + "overleaf_template_gallery": "", "overleafs_functionality_meets_my_needs": "", "overview": "", "overwrite": "", @@ -1340,6 +1353,7 @@ "please_ask_the_project_owner_to_upgrade_to_track_changes": "", "please_change_primary_to_remove": "", "please_compile_pdf_before_download": "", + "please_compile_pdf_before_publish_as_template": "", "please_confirm_primary_email_or_edit": "", "please_confirm_secondary_email_or_edit": "", "please_confirm_your_email_before_making_it_default": "", @@ -1375,6 +1389,7 @@ "premium_plan_label": "", "presentation_mode": "", "previous": "", + "prev": "", "previous_page": "", "price": "", "primarily_work_study_question": "", @@ -1872,7 +1887,10 @@ "tax_id_type": "", "tell_the_project_owner_and_ask_them_to_upgrade": "", "template": "", + "template_category": "", "template_description": "", + "template_gallery": "", + "template_title": "", "template_title_taken_from_project_title": "", "templates": "", "temporarily_hides_the_preview": "", diff --git a/services/web/frontend/js/features/editor-left-menu/components/actions-manage-template.tsx b/services/web/frontend/js/features/editor-left-menu/components/actions-manage-template.tsx new file mode 100644 index 00000000000..30d1e81eea3 --- /dev/null +++ b/services/web/frontend/js/features/editor-left-menu/components/actions-manage-template.tsx @@ -0,0 +1,72 @@ +import { useCallback, useState } from 'react' +import { useTranslation } from 'react-i18next' +import * as eventTracking from '../../../infrastructure/event-tracking' +import getMeta from '../../../utils/meta' +import OLTooltip from '@/features/ui/components/ol/ol-tooltip' +import { useDetachCompileContext } from '../../../shared/context/detach-compile-context' +import EditorManageTemplateModalWrapper from '../../manage-template-modal/components/editor-manage-template-modal-wrapper' +import LeftMenuButton from './left-menu-button' + +type TemplateManageResponse = { + template_id: string +} + +export default function ActionsManageTemplate() { + + const templatesAdmin = getMeta('ol-showTemplatesServerPro') + if (!templatesAdmin) { + return null + } + + const [showModal, setShowModal] = useState(false) + const { pdfFile } = useDetachCompileContext() + const { t } = useTranslation() + + const handleShowModal = useCallback(() => { + eventTracking.sendMB('left-menu-template') + setShowModal(true) + }, []) + + const openTemplate = useCallback( + ({ template_id: templateId }: TemplateManageResponse) => { + location.assign(`/template/${templateId}`) + }, + [location] + ) + + return ( + <> + {pdfFile ? ( + + {t('publish_as_template')} + + ) : ( + + {/* OverlayTrigger won't fire unless the child is a non-react html element (e.g div, span) */} +
+ + {t('publish_as_template')} + +
+ + )} + setShowModal(false)} + openTemplate={openTemplate} + /> + + ) +} diff --git a/services/web/frontend/js/features/manage-template-modal/components/editor-manage-template-modal-wrapper.jsx b/services/web/frontend/js/features/manage-template-modal/components/editor-manage-template-modal-wrapper.jsx new file mode 100644 index 00000000000..e24cae08214 --- /dev/null +++ b/services/web/frontend/js/features/manage-template-modal/components/editor-manage-template-modal-wrapper.jsx @@ -0,0 +1,37 @@ +import React from 'react' +import PropTypes from 'prop-types' +import withErrorBoundary from '../../../infrastructure/error-boundary' +import { useProjectContext } from '../../../shared/context/project-context' +import ManageTemplateModal from './manage-template-modal' + +const EditorManageTemplateModalWrapper = React.memo( + function EditorManageTemplateModalWrapper({ show, handleHide, openTemplate }) { + const { + _id: projectId, + name: projectName, + } = useProjectContext() + + if (!projectName) { + // wait for useProjectContext + return null + } else { + return ( + + ) + } + } +) + +EditorManageTemplateModalWrapper.propTypes = { + show: PropTypes.bool.isRequired, + handleHide: PropTypes.func.isRequired, + openTemplate: PropTypes.func.isRequired, +} + +export default withErrorBoundary(EditorManageTemplateModalWrapper) diff --git a/services/web/frontend/js/features/manage-template-modal/components/manage-template-modal-content.jsx b/services/web/frontend/js/features/manage-template-modal/components/manage-template-modal-content.jsx new file mode 100644 index 00000000000..54b3da3523e --- /dev/null +++ b/services/web/frontend/js/features/manage-template-modal/components/manage-template-modal-content.jsx @@ -0,0 +1,203 @@ +import { useMemo, useEffect, useState } from 'react' +import { useTranslation } from 'react-i18next' +import PropTypes from 'prop-types' +import { debugConsole } from '@/utils/debugging' +import { getJSON, postJSON } from '../../../infrastructure/fetch-json' +import Notification from '@/shared/components/notification' +import { + OLModalBody, + OLModalFooter, + OLModalHeader, + OLModalTitle, +} from '@/features/ui/components/ol/ol-modal' +import OLForm from '@/features/ui/components/ol/ol-form' +import OLFormGroup from '@/features/ui/components/ol/ol-form-group' +import OLFormControl from '@/features/ui/components/ol/ol-form-control' +import OLFormLabel from '@/features/ui/components/ol/ol-form-label' +import OLButton from '@/features/ui/components/ol/ol-button' +import { useDetachCompileContext } from '../../../shared/context/detach-compile-context' +import { useUserContext } from '../../../shared/context/user-context' +import SettingsTemplateCategory from './settings-template-category' + +const defaultLicense = 'Creative Commons CC BY 4.0' + +export default function ManageTemplateModalContent({ + handleHide, + inFlight, + setInFlight, + handleAfterPublished, + projectId, + projectName, +}) { + const { t } = useTranslation() + + const { pdfFile } = useDetachCompileContext() + const user = useUserContext() + + const [error, setError] = useState() + const [disablePublish, setDisablePublish] = useState(false) + const [notificationType, setNotificationType] = useState('error') + const [name, setName] = useState(`${projectName}`) + const [description, setDescription] = useState('') + const [author, setAuthor] = useState(`${user.first_name} ${user.last_name}`.trim()) + const [license, setLicense] = useState(defaultLicense) + const [category, setCategory] = useState() + const [override, setOverride] = useState(false) + const [titleConflict, setTitleConflict] = useState(false) + + const valid = useMemo( + () => name.trim().length > 0 && license.trim().length, + [name, license] + ) + + useEffect(() => { + const queryParams = new URLSearchParams({ key: 'name', val: projectName }) + getJSON(`/api/template?${queryParams}`) + .then((data) => { + if (!data) return + setDescription(data.descriptionMD) + setAuthor(data.authorMD) + setLicense(data.license) + setCategory(data.category) + }) + .catch(debugConsole.error) + }, []) + + const handleSubmit = event => { + event.preventDefault() + + if (!valid) { + return + } + + setError(false) + setInFlight(true) + + postJSON(`/template/new/${projectId}`, { + body: { + category, + name: name.trim(), + authorMD: author.trim(), + license: license.trim(), + descriptionMD: description.trim(), + build: pdfFile.build, + override, + }, + }) + .then(data => { + // redirect to template page + handleHide() + handleAfterPublished(data) + }) + .catch(({ response, data }) => { + if (response?.status === 409 && data.canOverride) { + setNotificationType('warning') + setOverride(true) + } else { + setNotificationType('error') + setDisablePublish(true) + } + setError(data.message) + if (response?.status === 409) setTitleConflict(true) + }) + .finally(() => { + setInFlight(false) + }) + } + + const handleNameChange = event => { + setName(event.target.value) + if (titleConflict) { + setError(false) + setOverride(false) + if (disablePublish) setDisablePublish(false) + } + } + + return ( + <> + + {t('publish_as_template')} + + + + + + {t('template_title')} + + + + + + + {t('Author')} + setAuthor(event.target.value)} + /> + + + {t('License')} + setLicense(event.target.value)} + /> + + + {t('template_description')} + setDescription(event.target.value)} + rows={4} + autoFocus + /> + + + {error && ( + + )} + + + + + {t('cancel')} + + + {inFlight ? <>{t('publishing')}… : override ? t('overwrite') : t('publish')} + + + + ) +} + +ManageTemplateModalContent.propTypes = { + handleHide: PropTypes.func.isRequired, + inFlight: PropTypes.bool, + handleAfterPublished: PropTypes.func.isRequired, + setInFlight: PropTypes.func.isRequired, + projectId: PropTypes.string, + projectName: PropTypes.string, +} diff --git a/services/web/frontend/js/features/manage-template-modal/components/manage-template-modal.jsx b/services/web/frontend/js/features/manage-template-modal/components/manage-template-modal.jsx new file mode 100644 index 00000000000..2e8096961d4 --- /dev/null +++ b/services/web/frontend/js/features/manage-template-modal/components/manage-template-modal.jsx @@ -0,0 +1,51 @@ +import React, { memo, useCallback, useState } from 'react' +import PropTypes from 'prop-types' +import OLModal from '@/features/ui/components/ol/ol-modal' +import ManageTemplateModalContent from './manage-template-modal-content' + +function ManageTemplateModal({ + show, + handleHide, + handleAfterPublished, + projectId, + projectName, +}) { + const [inFlight, setInFlight] = useState(false) + + const onHide = useCallback(() => { + if (!inFlight) { + handleHide() + } + }, [handleHide, inFlight]) + + return ( + + + + ) +} + +ManageTemplateModal.propTypes = { + show: PropTypes.bool.isRequired, + handleHide: PropTypes.func.isRequired, + handleAfterPublished: PropTypes.func.isRequired, + projectId: PropTypes.string, + projectName: PropTypes.string, +} + +export default memo(ManageTemplateModal) diff --git a/services/web/frontend/js/features/manage-template-modal/components/settings-template-category.tsx b/services/web/frontend/js/features/manage-template-modal/components/settings-template-category.tsx new file mode 100644 index 00000000000..747fbe73cf4 --- /dev/null +++ b/services/web/frontend/js/features/manage-template-modal/components/settings-template-category.tsx @@ -0,0 +1,51 @@ +import { useMemo, useEffect } from 'react' +import { useTranslation } from 'react-i18next' +import getMeta from '../../../utils/meta' +import SettingsMenuSelect from '@/features/editor-left-menu/components/settings/settings-menu-select' +import type { Option } from '@/features/editor-left-menu/components/settings/settings-menu-select' + +interface ManageTemplateCategoryProps { + category: string | null + setCategory: (value: string) => void +} + +export default function SettingsTemplateCategory({ + category, + setCategory +}: ManageTemplateCategoryProps) { + const { t } = useTranslation() + + const { templateLinks } = useMemo( + () => getMeta('ol-ExposedSettings') || [], + [] + ) + + if (templateLinks.length === 0) { + return null + } + + const options: Option[] = useMemo( + () => + templateLinks.map(({ name, url }) => ({ + value: url, + label: name, + })), + [templateLinks] + ) + + useEffect(() => { + if (!category && options.length > 0) { + setCategory(options[0].value) + } + }, [options, category, setCategory]) + + return ( + + ) +} diff --git a/services/web/frontend/js/features/template-gallery/components/gallery-header-all.tsx b/services/web/frontend/js/features/template-gallery/components/gallery-header-all.tsx new file mode 100644 index 00000000000..aed7c311f97 --- /dev/null +++ b/services/web/frontend/js/features/template-gallery/components/gallery-header-all.tsx @@ -0,0 +1,29 @@ +import { useTranslation } from 'react-i18next' +import OLCol from '@/features/ui/components/ol/ol-col' +import OLRow from '@/features/ui/components/ol/ol-row' + +export default function GalleryHeaderAll() { + const { t } = useTranslation() + return ( +
+ + +

+ + + {t('overleaf_template_gallery')} + + + {t('latex_templates')} +

+ + +
+
+

{t('latex_templates_for_journal_articles')} +

+
+
+
+ ) +} diff --git a/services/web/frontend/js/features/template-gallery/components/gallery-header-tagged.tsx b/services/web/frontend/js/features/template-gallery/components/gallery-header-tagged.tsx new file mode 100644 index 00000000000..ca2f37dbb56 --- /dev/null +++ b/services/web/frontend/js/features/template-gallery/components/gallery-header-tagged.tsx @@ -0,0 +1,33 @@ +import getMeta from '@/utils/meta' +import OLCol from '@/features/ui/components/ol/ol-col' +import OLRow from '@/features/ui/components/ol/ol-row' +import GallerySearchSortHeader from './gallery-search-sort-header' + +export default function GalleryHeaderTagged({ category }) { + const title = getMeta('og:title') + const { templateLinks } = getMeta('ol-ExposedSettings') || [] + + const description = templateLinks?.find(link => link.url.split("/").pop() === category)?.description + const gotoAllLink = (category !== 'all') + return ( +
+ + { category && ( + <> + + +

{title}

+ + + + +

{description}

+ + + + )} +
+ ) +} diff --git a/services/web/frontend/js/features/template-gallery/components/gallery-popular-tags.tsx b/services/web/frontend/js/features/template-gallery/components/gallery-popular-tags.tsx new file mode 100644 index 00000000000..9ab098ddabc --- /dev/null +++ b/services/web/frontend/js/features/template-gallery/components/gallery-popular-tags.tsx @@ -0,0 +1,29 @@ +import { useTranslation } from 'react-i18next' +import getMeta from '@/utils/meta' + +export default function GalleryPopularTags() { + const { t } = useTranslation() + const { templateLinks } = getMeta('ol-ExposedSettings') || [] + + return ( +
+

{t('categories')}

+
+ {templateLinks?.filter(link => link.url.split("/").pop() !== "all").map((link, index) => ( +
+ +
+ {link.name} +
+ {link.name} + +

{link.description}

+
+ ))} +
+
+ ) +} diff --git a/services/web/frontend/js/features/template-gallery/components/gallery-search-sort-header.tsx b/services/web/frontend/js/features/template-gallery/components/gallery-search-sort-header.tsx new file mode 100644 index 00000000000..8fe9438a821 --- /dev/null +++ b/services/web/frontend/js/features/template-gallery/components/gallery-search-sort-header.tsx @@ -0,0 +1,78 @@ +import { useTemplateGalleryContext } from '../context/template-gallery-context' +import { useTranslation } from 'react-i18next' +import SearchForm from './search-form' +import OLCol from '@/features/ui/components/ol/ol-col' +import OLRow from '@/features/ui/components/ol/ol-row' +import useSort from '../hooks/use-sort' +import withContent, { SortBtnProps } from './sort/with-content' +import MaterialIcon from '@/shared/components/material-icon' + +function SortBtn({ onClick, text, iconType, screenReaderText }: SortBtnProps) { + return ( + + ) +} + +const SortByButton = withContent(SortBtn) + +export default function GallerySearchSortHeader( { gotoAllLink }: { boolean } ) { + const { t } = useTranslation() + const { + searchText, + setSearchText, + sort, + } = useTemplateGalleryContext() + + const { handleSort } = useSort() + return ( + + {gotoAllLink ? ( + + + + {t('all_templates')} + + + ) : ( + + + + {t('template_gallery')} + + + )} + + handleSort('lastUpdated')} + /> + + handleSort('name')} + /> + + + + + + ) +} diff --git a/services/web/frontend/js/features/template-gallery/components/pagination.tsx b/services/web/frontend/js/features/template-gallery/components/pagination.tsx new file mode 100644 index 00000000000..a55f96a6a49 --- /dev/null +++ b/services/web/frontend/js/features/template-gallery/components/pagination.tsx @@ -0,0 +1,80 @@ +import { useTranslation } from 'react-i18next' + +export default function Pagination({ currentPage, totalPages, onPageChange }) { + const { t } = useTranslation() + if (totalPages <= 1) return null + + const pageNumbers = [] + let startPage = Math.max(1, currentPage - 4) + let endPage = Math.min(totalPages, currentPage + 4) + + if (startPage > 1) { + pageNumbers.push(1) + if (startPage > 2) { + pageNumbers.push("...") + } + } + + for (let i = startPage; i <= endPage; i++) { + pageNumbers.push(i) + } + + if (endPage < totalPages) { + if (endPage < totalPages - 1) { + pageNumbers.push("...") + } + pageNumbers.push(totalPages) + } + + return ( + + ) +} diff --git a/services/web/frontend/js/features/template-gallery/components/search-form.tsx b/services/web/frontend/js/features/template-gallery/components/search-form.tsx new file mode 100644 index 00000000000..cb05a91e951 --- /dev/null +++ b/services/web/frontend/js/features/template-gallery/components/search-form.tsx @@ -0,0 +1,59 @@ +import { useTranslation } from 'react-i18next' +import { MergeAndOverride } from '../../../../../types/utils' +import OLForm from '@/features/ui/components/ol/ol-form' +import OLFormControl from '@/features/ui/components/ol/ol-form-control' +import MaterialIcon from '@/shared/components/material-icon' + +type SearchFormOwnProps = { + inputValue: string + setInputValue: (input: string) => void +} + +type SearchFormProps = MergeAndOverride< + React.ComponentProps, + SearchFormOwnProps +> + +export default function SearchForm({ + inputValue, + setInputValue, +}: SearchFormProps) { + const { t } = useTranslation() + let placeholderMessage = t('search') + const placeholder = `${placeholderMessage}…` + + const handleChange: React.ComponentProps['onChange'] = e => { + setInputValue(e.target.value) + } + + const handleClear = () => setInputValue('') + + return ( + e.preventDefault()} + > + } + append={ + inputValue.length > 0 && ( + + ) + } + /> + + ) +} diff --git a/services/web/frontend/js/features/template-gallery/components/sort/with-content.tsx b/services/web/frontend/js/features/template-gallery/components/sort/with-content.tsx new file mode 100644 index 00000000000..8c77484fdf2 --- /dev/null +++ b/services/web/frontend/js/features/template-gallery/components/sort/with-content.tsx @@ -0,0 +1,46 @@ +import { useTranslation } from 'react-i18next' +import { Sort } from '../../types/api' + +type SortBtnOwnProps = { + column: string + sort: Sort + text: string + onClick: () => void +} + +type WithContentProps = { + iconType?: string + screenReaderText: string +} + +export type SortBtnProps = SortBtnOwnProps & WithContentProps + +function withContent( + WrappedComponent: React.ComponentType +) { + function WithContent(hocProps: T) { + const { t } = useTranslation() + const { column, text, sort } = hocProps + let iconType + + let screenReaderText = t('sort_by_x', { x: text }) + + if (column === sort.by) { + iconType = + sort.order === 'asc' ? 'arrow_upward_alt' : 'arrow_downward_alt' + screenReaderText = t('reverse_x_sort_order', { x: text }) + } + + return ( + + ) + } + + return WithContent +} + +export default withContent diff --git a/services/web/frontend/js/features/template-gallery/components/template-gallery-entry.tsx b/services/web/frontend/js/features/template-gallery/components/template-gallery-entry.tsx new file mode 100644 index 00000000000..479b306b51d --- /dev/null +++ b/services/web/frontend/js/features/template-gallery/components/template-gallery-entry.tsx @@ -0,0 +1,29 @@ +import { memo } from 'react' +import { cleanHtml } from '../../../../../modules/template-gallery/app/src/CleanHtml.mjs' + +function TemplateGalleryEntry({ template }) { + return ( +
+ +
+ {template.name} +
+ + {template.name} + + + +
+

+

+
+
+
+
+ ) +} + +export default memo(TemplateGalleryEntry) diff --git a/services/web/frontend/js/features/template-gallery/components/template-gallery-root.tsx b/services/web/frontend/js/features/template-gallery/components/template-gallery-root.tsx new file mode 100644 index 00000000000..d17250f33a5 --- /dev/null +++ b/services/web/frontend/js/features/template-gallery/components/template-gallery-root.tsx @@ -0,0 +1,64 @@ +import { TemplateGalleryProvider } from '../context/template-gallery-context' +import { useTranslation } from 'react-i18next' +import useWaitForI18n from '../../../shared/hooks/use-wait-for-i18n' +import withErrorBoundary from '../../../infrastructure/error-boundary' +import { GenericErrorBoundaryFallback } from '@/shared/components/generic-error-boundary-fallback' +import getMeta from '@/utils/meta' +import DefaultNavbar from '@/features/ui/components/bootstrap-5/navbar/default-navbar' +import Footer from '@/features/ui/components/bootstrap-5/footer/footer' +import GalleryHeaderTagged from './gallery-header-tagged' +import GalleryHeaderAll from './gallery-header-all' +import TemplateGallery from './template-gallery' +import GallerySearchSortHeader from './gallery-search-sort-header' +import GalleryPopularTags from './gallery-popular-tags' + +function TemplateGalleryRoot() { + const { isReady } = useWaitForI18n() + if (!isReady) { + return null + } + return ( + + + + ) +} + +function TemplateGalleryPageContent() { + const { t } = useTranslation() + const navbarProps = getMeta('ol-navbar') + const footerProps = getMeta('ol-footer') + const category = getMeta('ol-templateCategory') + + return ( + <> + +
+
+ {category ? ( + <> + + + + ) : ( + <> + + +
+
+ +

{t('all_templates')}

+ +
+ + )} +
+
+