Document Processing

2. Process PDF Files

☐ Install PDF.js library (pdfjs-dist) ☐ Configure worker for PDF parsing ☐ Extract text from PDF pages ☐ Handle PDF rendering (if showing preview)

// lib/pdfProcessor.ts
import * as pdfjsLib from 'pdfjs-dist';

// Configure worker
pdfjsLib.GlobalWorkerOptions.workerSrc = `//cdnjs.cloudflare.com/ajax/libs/pdf.js/${pdfjsLib.version}/pdf.worker.min.js`;

export async function extractTextFromPDF(file: File): Promise<string> {
  const arrayBuffer = await file.arrayBuffer();
  const pdf = await pdfjsLib.getDocument({ data: arrayBuffer }).promise;

  let fullText = '';

  for (let i = 1; i <= pdf.numPages; i++) {
    const page = await pdf.getPage(i);
    const textContent = await page.getTextContent();
    const pageText = textContent.items
      .map((item: any) => item.str)
      .join(' ');

    fullText += pageText + '\n\n';
  }

  return fullText.trim();
}

3. Process DOCX Files

☐ Install mammoth library (mammoth) ☐ Extract text or HTML from DOCX ☐ Handle formatting preservation (if needed) ☐ Convert to plain text or structured format

// lib/docxProcessor.ts
import mammoth from 'mammoth';

export async function extractTextFromDOCX(file: File): Promise<string> {
  const arrayBuffer = await file.arrayBuffer();

  const result = await mammoth.extractRawText({ arrayBuffer });

  if (result.messages.length > 0) {
    console.warn('DOCX conversion warnings:', result.messages);
  }

  return result.value;
}

export async function extractHTMLFromDOCX(file: File): Promise<string> {
  const arrayBuffer = await file.arrayBuffer();

  const result = await mammoth.convertToHtml({ arrayBuffer });

  return result.value;
}

4. Create Document Processing Hook

☐ Create hook to handle multiple file types ☐ Implement processing logic for each type ☐ Handle loading and error states ☐ Return extracted content

// hooks/useDocumentProcessor.ts
import { useState } from 'react';
import { extractTextFromPDF } from '../lib/pdfProcessor';
import { extractTextFromDOCX } from '../lib/docxProcessor';

export function useDocumentProcessor() {
  const [processing, setProcessing] = useState(false);
  const [error, setError] = useState<string | null>(null);

  const processDocument = async (file: File): Promise<string | null> => {
    setProcessing(true);
    setError(null);

    try {
      const fileType = file.name.split('.').pop()?.toLowerCase();

      let text: string;

      switch (fileType) {
        case 'pdf':
          text = await extractTextFromPDF(file);
          break;
        case 'docx':
          text = await extractTextFromDOCX(file);
          break;
        case 'txt':
          text = await file.text();
          break;
        default:
          throw new Error(`Unsupported file type: ${fileType}`);
      }

      return text;
    } catch (err) {
      const message = err instanceof Error ? err.message : 'Processing failed';
      setError(message);
      return null;
    } finally {
      setProcessing(false);
    }
  };

  return { processDocument, processing, error };
}

5. Implement Upload to Backend

☐ Create FormData for file upload ☐ Send file to backend API ☐ Track upload progress ☐ Handle backend response

// lib/uploadDocument.ts
export async function uploadDocument(
  file: File,
  onProgress?: (progress: number) => void
): Promise<{ id: string; url: string }> {
  const formData = new FormData();
  formData.append('file', file);

  return new Promise((resolve, reject) => {
    const xhr = new XMLHttpRequest();

    // Track upload progress
    xhr.upload.addEventListener('progress', (e) => {
      if (e.lengthComputable && onProgress) {
        const progress = (e.loaded / e.total) * 100;
        onProgress(progress);
      }
    });

    xhr.addEventListener('load', () => {
      if (xhr.status === 200) {
        resolve(JSON.parse(xhr.responseText));
      } else {
        reject(new Error(`Upload failed: ${xhr.statusText}`));
      }
    });

    xhr.addEventListener('error', () => {
      reject(new Error('Network error during upload'));
    });

    xhr.open('POST', '/api/documents/upload');
    xhr.send(formData);
  });
}

6. Verify Document Processing Works

☐ Test with various file types (PDF, DOCX, TXT) ☐ Test with large files (check performance) ☐ Test error cases (invalid files, corrupted documents) ☐ Verify text extraction accuracy ☐ Test upload to backend (if applicable)

Common Pitfalls

Not configuring PDF.js worker:

// ❌ Wrong - worker not configured
const pdf = await pdfjsLib.getDocument({ data }).promise;

// ✅ Right - configure worker first
pdfjsLib.GlobalWorkerOptions.workerSrc = '...';
const pdf = await pdfjsLib.getDocument({ data }).promise;

Reading file incorrectly:

// ❌ Wrong - file content not read
const text = file.name;

// ✅ Right - read file content
const text = await file.text(); // For text files
const buffer = await file.arrayBuffer(); // For binary files

Red Flags

Never:

• Process files without size validation (memory issues)
• Skip error handling for corrupted files
• Load entire large files into memory at once
• Trust user-provided file types (validate with magic numbers)

Always:

• Validate file size before processing
• Handle processing errors gracefully
• Show progress indicators for large files
• Use web workers for heavy processing (keep UI responsive)
• Sanitize extracted content before displaying
• Implement file type validation (not just extension)