Type: AWS::S3::Bucket
Properties:
  BucketName: serverless-search-${aws:accountId}-${aws:region}
  NotificationConfiguration:
    EventBridgeConfiguration:
      EventBridgeEnabled: true
  

  
handler: src/handlers/indexBuild.handler
name: ${self:service}-index-build
memorySize: 5120
timeout: 900
environment:
  BUCKET:
    Ref: S3BucketSearch
events:
  - eventBridge:
      pattern:
        source:
          - aws.s3
        detail-type:
          - Object Created
        detail:
          bucket:
            name:
              - Ref: S3BucketSearch
          object:
            key:
              - prefix: 'source/'
  

  
import internal from 'stream'
import {
  GetObjectCommand,
  PutObjectCommand,
  PutObjectCommandInput,
  S3Client,
} from '@aws-sdk/client-s3'
import csv from 'csvtojson'
import lunr from 'lunr'

const bucket = process.env.BUCKET
const indexPrefix = 'indexes/'

export const handler = async (event: S3ObjectCreatedNotificationEvent): Promise => {
  const key = event.detail.object.key

  const sourceName = path.basename(key, path.extname(key))

  const command = new GetObjectCommand({
    Bucket: params.bucket,
    Key: params.key,
  })
  const res = await client.send(command)
  if (!(res.Body instanceof internal.Readable)) {
    return
  }

  const sourceStream = res.Body

  let fields: string[] = []
  const allData = await csv()
    .on('header', (header) => {
      fields = header
    })
    .fromStream(sourceStream)

  const idx: lunr.Index = lunr(function () {
    this.ref(fields[0])
    fields.slice(1).forEach((f) => {
      this.field(f)
    })

    params.documents.forEach((document) => {
      this.add(document)
    })
  })

  await Promise.all([
    store({
      bucket,
      content: JSON.stringify(idx.toJSON()),
      filename: sourceName,
      fileType: 'idx',
    }),
    store({
      bucket,
      content: JSON.stringify(allData),
      filename: sourceName,
      fileType: 'doc',
    })
  ])
}

const store = async (params: Store): Promise => {
  const s3Params: PutObjectCommandInput = {
    Bucket: params.bucket,
    Key: `${indexPrefix}${params.filename}/${params.fileType}`,
    ContentType: 'application/json',
    Body: params.content,
  }
  const command = new PutObjectCommand(s3Params)
  return client.send(command)
}
  

  
const bucket = process.env.BUCKET

let idx
let documents

export const handler = async (
  event: APIGatewayProxyEventV2,
): Promise> => {
  const searchQuery = event.queryStringParameters.search
  const indexName = event.queryStringParameters.indexname

  if (!idx) {
    idx = lunr.Index.load(JSON.parse(await getFromS3({ bucket, filename: indexName, fileType: 'idx' })))
    documents = JSON.parse(await getFromS3({ bucket, filename: indexName, fileType: 'doc' }))
  }

  const queryResult = idx.search(searchQuery)

  const response = queryResult.map((item) => {
    const match = documents.find((doc) => item.ref === doc.set_id)
    if (match) {
      const enhancedMatch: SearchResponseItem = {
        ...match,
        matchInfo: {
          score: item.score,
          matchData: item.matchData,
        },
      }
      return enhancedMatch
    }
    return undefined
  })

  return response
}
  

‍

Query performance

• Search Lego sets produced in 1984 in the Duplo theme

              •   query: 'search=+year:1984 +theme:duplo'

              •   amount of results: 17

              •   loading index and documents: 1927 ms (only on first load)

              •   query duration: 10 ms

              •   memory used: 562 MB

• Search Movies with Ryan Reynolds

              •   query: 'search=+ryan +reynolds'

              •   amount of results: 37

              •   loading index and documents: 3924 ms (only on first load)

              •   query duration: 5 ms

              •   memory used: 690 MB

• Search Movies with the word "extra-terrestrial" in the synopsis

              •   query: 'search=overview:extra-terrestrial'

              •   amount of results: 17

              •   loading index and documents: 3411 ms (only on first load)

              •   query duration: 4 ms

              •   memory used: 690 MB

As we can notice, loading the index is very slow, but twice as fast as building it. Requests using an already loaded function, don't have any latency penalty.

Cost analysis

Let's consider the movie dataset. And make the following assumptions:

• Source file updated daily (30 times a month)
• 1M search requests per month

              •   25% are fresh: 250,000

              •   75% are re-using an already loaded Lambda: 750,000

Daniel Muller

Senior Serverless Developer at ServerlessGuru

Daniel has has developed multiple large-scale serverless applications for OTT, BigData, and MarTech and is currently a Sr Serverless Dev.