Llama 3.1 70B: Run Locally with Ollama (2026 Guide)

import { Ollama } from 'ollama';

const ollama = new Ollama({ host: 'http://localhost:11434' });

// Basic chat
async function chat(userMessage: string) {
  const response = await ollama.chat({
    model: 'llama3.1:70b',
    messages: [{ role: 'user', content: userMessage }],
  });
  return response.message.content;
}

// Streaming response (for real-time UI)
async function streamChat(userMessage: string) {
  const response = await ollama.chat({
    model: 'llama3.1:70b',
    messages: [{ role: 'user', content: userMessage }],
    stream: true,
  });
  for await (const part of response) {
    process.stdout.write(part.message.content);
  }
}

// RAG: Feed a long document into the 128K context
async function analyzeDocument(filePath: string, question: string) {
  const fs = await import('fs/promises');
  const document = await fs.readFile(filePath, 'utf-8');

  const response = await ollama.generate({
    model: 'llama3.1:70b',
    prompt: `Based on this document, answer: ${question}\n\nDocument:\n${document}`,
    options: { temperature: 0.3, num_ctx: 131072 },
  });
  return response.response;
}

// REST API wrapper (Express)
import express from 'express';
const app = express();
app.use(express.json());

app.post('/api/chat', async (req, res) => {
  const { message } = req.body;
  const answer = await chat(message);
  res.json({ answer });
});

app.listen(3000, () => console.log('Llama 70B API on :3000'));