roam-research-private-api/EvernoteSync.js

Library that loads your Roam Research graph as a browser and performs tasks as you.

const Evernote = require( 'evernote' );
const ENML = require( 'enml-js' );
const RoamSyncAdapter = require( './Sync' );
const moment = require( 'moment' );

class EvernoteSyncAdapter extends RoamSyncAdapter {
	EvernoteClient = null;
	NoteStore = null;
	notebookGuid = '';
	defaultNotebook = '';
	timeOut = 500;
	mapping;
	backlinks = {};
	notesBeingImported = [];

	wrapItem( string, title ) {
		return `<li>${ string }</li>`;
	}
	wrapText( string, title ) {
		const backlinks = [];
		string = this.htmlEntities( string );
		string = string.replace( '{{[[TODO]]}}', '[[TODO]]' ); // <en-todo/> will not achieve the same goal.
		string = string.replace( '{{{[[DONE]]}}}}', '<en-todo checked="true"/>' );
		string = string.replace( /\!\[([^\]]*?)\]\(([^\)]+)\)/g, '<img src="$2"/>' );
		string = string.replace( /\[([^\]]+)\]\((http|evernote)([^\)]+)\)/g, '<a href="$2$3">$1</a>' );
		string = string.replace( /(^|[^"?/])((evernote|http|https|mailto):[a-zA-Z0-9\/.\?\&=;\-_]+)/g, '$1<a href="$2">$2</a>' );
		string = string.replace( /\*\*([^*]+)\*\*/g, '<b>$1</b>' );
		string = string.replace( /__([^_]+)__/g, '<i>$1</i>' );
		string = string.replace( /#?\[\[([^\]]+)\]\]/g, ( match, contents ) => {
			const targetPage = this.titleMapping.get( contents );
			if (
				targetPage &&
				targetPage.uid &&
				this.mapping.get( targetPage.uid )
			) {
				const guid = this.mapping.get( targetPage.uid ).guid;
				const url = this.getNoteUrl( guid );
				backlinks.push( contents );
				return `<a href="${ url }">${ contents }</a>`;
			}
			return match;
		} );
		this.addBacklink( backlinks, title, string );
		return string;
	}

	wrapChildren( childrenString ) {
		childrenString = childrenString.join( '' );
		return `<ul>${ childrenString }</ul>`;
	}
	htmlEntities( str ) {
		return String( str )
			.replace( /&/g, '&amp;' )
			.replace( /</g, '&lt;' )
			.replace( />/g, '&gt;' )
			.replace( /"/g, '&quot;' );
	}
	htmlEntitiesDecode( str ) {
		return String( str )
			.replace( '&amp;', '&' )
			.replace( '&lt;', '<' )
			.replace( '&gt;', '>' )
			.replace( '&quot;', '"' );
	}
	wrapNote( noteBody ) {
		noteBody = noteBody.replace( '&Amp;', '&amp;' ); // Readwise artifact
		var nBody = '<?xml version="1.0" encoding="UTF-8"?>';
		nBody += '<!DOCTYPE en-note SYSTEM "http://xml.evernote.com/pub/enml2.dtd">';
		nBody += '<en-note>' + noteBody;
		nBody += '</en-note>';
		return nBody;
	}

	makeNote( noteTitle, noteBody, url, uid ) {
		// Create note object
		var nBody = this.wrapNote( noteBody );
		let foundNote;

		if ( this.mapping.get( uid ) ) {
			foundNote = Promise.resolve( { 
				totalNotes: 1,
				notes: [ this.mapping.get( uid ) ]
			} );
		} else {
			foundNote = this.findPreviousNote( url );
		}

		const foundNotes = ( notes ) => {
			if ( ! notes.totalNotes ) {
				return Promise.resolve( new Evernote.Types.Note() );
			}
			if ( notes.notes[0].contentLength === nBody.length ) {
				//console.log( '[[' + noteTitle + ']]: content length has not changed, skipping' );
				return Promise.resolve( false );
			}
			// These request we want to rate limit.
			return new Promise( ( resolve, reject ) => setTimeout( () => {
				this.NoteStore.getNote( notes.notes[0].guid, true, false, false, false )
					.catch( ( err ) => {
						console.warn( '[[' + noteTitle + ']] :Took too long to pull note ' );
						if ( err.code === 'ETIMEDOUT' ) {
							this.timeOut = this.timeOut * 2;
							console.log( 'Exponential Backoff increased: ', this.timeOut );
						}
						if ( this.timeOut > 60000 * 5 ) {
							console.error( 'Timeot reached 5 minutes. Exiting' );
							require('process').exit();
						}
						resolve( foundNotes( notes ) );
					} )
					.then( result => resolve( result ) );
			}, this.timeOut ) );

		}

		foundNote.then( foundNotes )
		.then( ( ourNote ) => {
			if ( ! ourNote ) {
				return Promise.resolve( false );
			}
			// Build body of note
			if ( ourNote.content && ourNote.content === nBody ) {
				console.log( '[[' + noteTitle + ']]: has not changed, skipping' );
				return Promise.resolve( ourNote );
			}
			ourNote.content = nBody;

			const attributes = new Evernote.Types.NoteAttributes();
			attributes.contentClass = 'piszek.roam';
			attributes.sourceURL = url;
			attributes.sourceApplication = 'piszek.roam';
			ourNote.attributes = attributes;
			ourNote.title = this.htmlEntities( noteTitle.trim() );

			if ( ourNote.guid ) {
				console.log( '[[' + noteTitle + ']]: updating' );
				ourNote.updated = Date.now();
				return this.NoteStore.updateNote( ourNote ).catch( err => {
					console.log( 'Update note problem', err, nBody );
					return Promise.resolve( false );
				} );
			} else {
				// parentNotebook is optional; if omitted, default notebook is used
				if ( this.notebookGuid ) {
					ourNote.notebookGuid = this.notebookGuid;
				}
				console.log( '[[' + noteTitle + ']] Creating new note ' );
				return this.NoteStore.createNote( ourNote ).then( ( note ) => {
					this.mapping.set( uid, {
						guid: note.guid,
						title: note.title,
						contentLength: note.contentLength
					} );
					return Promise.resolve( note );
				} ).catch( err => {
					console.warn( 'Error creating note:', err );
					Promise.resolve( false );
				});
			}
		} );
	}

	findNotebook() {
		return new Promise( ( resolve, reject ) => {
			this.NoteStore.listNotebooks().then( ( notebooks ) => {
				const filtered = notebooks.filter( ( nb ) => nb.name === 'Roam' );
				const def = notebooks.filter( ( nb ) => nb.defaultNotebook );
				this.defaultNotebook = def[ 0 ].guid
				if ( filtered ) {
					this.notebookGuid = filtered[ 0 ].guid;
					resolve( this.notebookGuid );
				} else {
					console.warn( 'You have to have a notebook named "Roam"' );
					reject( 'You have to have a notebook named "Roam"' );
				}
			} );
		} );
	}
	getNotesToImport() {
		const filter = new Evernote.NoteStore.NoteFilter();
		const spec = new Evernote.NoteStore.NotesMetadataResultSpec();
		spec.includeTitle = false;
		filter.words = '-tag:RoamImported';
		filter.notebookGuid = this.defaultNotebook;
		const batchCount = 100;
		const loadMoreNotes = ( result ) => {
			if ( result.notes ) {
				this.notesBeingImported = this.notesBeingImported.concat(
					result.notes.map( ( note ) =>
						this.NoteStore.getNote( note.guid, true, false, false, false )
					)
				);
			}
			if ( result.startIndex < result.totalNotes ) {
				return this.NoteStore.findNotesMetadata(
					filter,
					result.startIndex + result.notes.length,
					batchCount,
					spec
				).then( loadMoreNotes );
			} else {
				return Promise.resolve( this.mapping );
			}
		};
		return this.NoteStore.findNotesMetadata( filter, 0, batchCount, spec )
			.then( loadMoreNotes )
			.then( () =>
				Promise.all( this.notesBeingImported ).then( ( notes ) => {
					this.notesBeingImported = notes;
					return Promise.resolve( notes );
				} )
			);
	}
	adjustTitle( title, force ) {
		if ( force || title === 'Bez tytułu' || title === 'Untitled Note' ) {
			return moment( new Date() ).format( 'MMMM Do, YYYY' );
		} else {
			return title;
		}
	}
	getRoamPayload() {
		return this.notesBeingImported.map( ( note ) => {
			const md = ENML.PlainTextOfENML( note.content );
			return {
				title: this.adjustTitle( note.title, true ),
				children: [ { string: 'Imported from Evernote: [' + note.title + '](' + this.getNoteUrl( note.guid ) + ')', children: [ { string: md } ] } ],
			};
		} );
	}
	cleanupImportNotes() {
		return Promise.all(
			this.notesBeingImported.map( ( note ) => {
				note.tagGuids = [];
				note.tagNames = [ 'RoamImported' ];
				return this.NoteStore.updateNote( note );
			} )
		);
	}

	findPreviousNote( url ) {
		const filter = new Evernote.NoteStore.NoteFilter();
		const spec = new Evernote.NoteStore.NotesMetadataResultSpec();
		spec.includeContentLength = true;
		spec.includeTitle = true;
		filter.words = `sourceUrl:"${url}" contentClass:piszek.roam`;
		return this.NoteStore.findNotesMetadata( filter, 0, 1, spec );
	}

	loadPreviousNotes() {
		let duplicates = 0;
		const filter = new Evernote.NoteStore.NoteFilter();
		const spec = new Evernote.NoteStore.NotesMetadataResultSpec();
		spec.includeTitle = true;
		spec.includeContentLength = true;
		spec.includeDeleted = false;
		spec.includeAttributes = true;
		filter.words = 'contentClass:piszek.roam';
		const batchCount = 100;

		const loadMoreNotes = ( result ) => {
			if ( result.notes ) {
				result.notes.forEach( ( note ) => {
					if ( ! note.attributes.sourceURL ) {
						console.log( note.title , 'no src url' );
						return;
					}
					const match = note.attributes.sourceURL.match( /https:\/\/roamresearch\.com\/\#\/app\/[a-z]+\/page\/([a-zA-Z0-9_-]+)/);
					if ( ! match ) {
						console.log( note.title ,'no match', note.attributes.sourceURL );
						return;
					}

					const uid = match[1];
						
					if ( this.mapping.get( uid ) && this.mapping.get( uid ).guid !== note.guid ) {
						console.log(
							'[[' + this.mapping.get( uid ).title + ']]',
							'Note is a duplicate ',
							this.getNoteUrl( this.mapping.get( uid ).guid ),
							this.getNoteUrl( note.guid )
						);
						// this.NoteStore.deleteNote( note.guid );
					} else {
						this.mapping.set( uid, {
							guid: note.guid,
							title: note.title,
							contentLength: note.contentLength
						} );
					}
				} );
			}
			if ( result.startIndex < result.totalNotes ) {
				return this.NoteStore.findNotesMetadata(
					filter,
					result.startIndex + result.notes.length,
					batchCount,
					spec
				).then( loadMoreNotes );
			} else {
				// console.log( batchCount );
				return Promise.resolve( this.mapping );
			}
		};
		return this.NoteStore.findNotesMetadata( filter, 0, batchCount, spec ).then( loadMoreNotes );
	}

	addBacklink( titles, target, text ) {
		titles.forEach( ( title ) => {
			if ( ! this.backlinks[ title ] ) {
				this.backlinks[ title ] = [];
			}
			this.backlinks[ title ].push( {
				target: target,
				text: text,
			} );
		} );
	}
	getNoteUrl( guid ) {
		return `evernote:///view/${ this.user.id }/${ this.user.shardId }/${ guid }/${ guid }/`;
	}
	init( prevData = {} ) {
		this.mapping = new Map( prevData );
		this.EvernoteClient = new Evernote.Client( this.credentials );
		this.NoteStore = this.EvernoteClient.getNoteStore();
		const loadNotesPromise = this.loadPreviousNotes();
		loadNotesPromise.then( () => console.log( 'Loaded previous notes: ', this.mapping.size ) );

		return Promise.all( [
			new Promise( ( resolve, reject ) => {
				this.EvernoteClient.getUserStore()
					.getUser()
					.then( ( user ) => {
						this.user = user;
						resolve();
					} );
			} ),
			this.findNotebook().catch( ( err ) => console.log( 'Cannot find notebook Roam:', err ) ),
			loadNotesPromise,
		] );
	}

	sync( pages ) {
		// This can potentially introduce a race condition, but it's unlikely. Famous last words.
		var p = Promise.resolve();
		pages.forEach( ( page ) => {
			p = p
				.then( () => this.syncPage( page ) )
				.catch( ( err ) => console.warn( 'Problem with syncing page ' + page.title, err, page.content ) );
		} );
		return p.then( () => Promise.resolve( this.mapping ) );
	}

	syncPage( page ) {
		let url;
		if ( page.uid ) {
			url = `https://roamresearch.com/#/app/${this.graphName}/page/` + page.uid;
		} else {
			console.warn( "Page must have UIDs and this one does not. Using title as a backup." );
			url = page.title;
		}
		let newContent = page.content;
		if ( this.backlinks[ page.title ] ) {
			const list = this.backlinks[ page.title ]
				.map( ( target ) => {
					let reference = '[[' + target.target + ']]';
					const targetPage = this.titleMapping.get( target.target );
					if (
						targetPage &&
						targetPage.uid &&
						this.mapping.get( targetPage.uid )
					) {
						reference =
							'<a href="' +
							this.getNoteUrl( this.mapping.get( targetPage.uid ).guid ) +
							'">' +
							target.target +
							'</a>';
					}
					return '<li>' + reference + ': ' + target.text + '</li>';
				} )
				.join( '' );

			const backlinks = '<h3>Linked References</h3><ul>' + list + '</ul>';
			newContent = page.content + backlinks;
		}

		return this.makeNote( page.title, newContent, url, page.uid );
	}
}

module.exports = EvernoteSyncAdapter;