Skip to content
Snippets Groups Projects
Commit ef25c3bf authored by Dean's avatar Dean
Browse files

update Elasticsearch mapping

parent a8eed0ae
No related branches found
No related tags found
No related merge requests found
...@@ -2,6 +2,10 @@ ...@@ -2,6 +2,10 @@
indent_style = tab indent_style = tab
indent_size = 8 indent_size = 8
[*.json]
indent_style = space
indent_size = 2
[*.sql] [*.sql]
indent_style = space indent_style = space
indent_size = 2 indent_size = 2
......
...@@ -45,16 +45,17 @@ Elaticsearch server in the following format: ...@@ -45,16 +45,17 @@ Elaticsearch server in the following format:
```js ```js
{ {
"country_code": string, "country_code": keyword,
"hostname": string, "hostname": keyword,
"object_type": string, "object_type": keyword,
"status_code": int, "status_code": short,
"@timestamp": date // generated from `timestamp` pipeline "@timestamp": date // generated from `@timestamp` pipeline
} }
``` ```
The index and `timestamp` pipeline are created automatically if `cdn-origin` has The index and `@timestamp` pipeline are created automatically if `cdn-origin`
permission. has permission. Alternatively, the mapping and pipeline can be created by other
means using the `.json` files in [metrics/](metrics).
### TODO ### TODO
......
...@@ -13,6 +13,8 @@ import ( ...@@ -13,6 +13,8 @@ import (
"gopkg.in/olivere/elastic.v5/config" "gopkg.in/olivere/elastic.v5/config"
) )
// mapping is the default mapping to use when creating the index if it doesn't exist. This JSON data is also maintained
// in `./mapping.elasticsearch.json`.
const mapping = ` const mapping = `
{ {
"settings": { "settings": {
...@@ -20,16 +22,39 @@ const mapping = ` ...@@ -20,16 +22,39 @@ const mapping = `
}, },
"mappings": { "mappings": {
"type": { "request": {
"properties": { "properties": {
"timestamp": { "country_code": {
"type": "date" "type": "keyword",
"ignore_above": 2,
"index": true
},
"hostname": {
"type": "keyword",
"ignore_above": 30,
"index": true
},
"object_type": {
"type": "keyword",
"ignore_above": 30,
"index": true
},
"status_code": {
"type": "short",
"index": true
},
"@timestamp": {
"type": "date",
"index": true
} }
} }
} }
} }
}` }`
// The default `@timestamp` pipeline. Sets the `@timestamp` field to the ingest timestamp (date type). This JSON data is
// also maintained in `./timestampPipeline.elasticsearch.json`.
const timestampPipeline = ` const timestampPipeline = `
{ {
"description": "Stores the ingest timestamp as a date field in the document.", "description": "Stores the ingest timestamp as a date field in the document.",
......
{
"settings": {
"number_of_shards": 1
},
"mappings": {
"request": {
"properties": {
"country_code": {
"type": "keyword",
"ignore_above": 2,
"index": true
},
"hostname": {
"type": "keyword",
"ignore_above": 30,
"index": true
},
"object_type": {
"type": "keyword",
"ignore_above": 30,
"index": true
},
"status_code": {
"type": "short",
"index": true
},
"@timestamp": {
"type": "date",
"index": true
}
}
}
}
}
{
"description": "Stores the ingest timestamp as a date field in the document.",
"processors": [
{
"set": {
"field": "@timestamp",
"value": "{{_ingest.timestamp}}"
}
},
{
"date": {
"field": "@timestamp",
"target_field": "@timestamp",
"formats": ["EEE MMM d HH:mm:ss z yyyy"]
}
}
]
}
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment