Elgato_dark/fleet
1
0
Fork 0
mirror of https://github.com/fleetdm/fleet synced 2026-05-23 00:49:03 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 106

Remove sandbox infrastructure directory (#16902)

Browse source
This commit is contained in:
Luke Heath 2024-02-21 16:07:58 -06:00 committed by GitHub
parent 00cd829dab
commit 186fdec05a
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
272 changed files with 0 additions and 33510 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
infrastructure/sandbox/.gitignore vendored
View file

@ -1,2 +0,0 @@
.external_modules
.tfsec

342
infrastructure/sandbox/.terraform.lock.hcl
View file

@ -1,342 +0,0 @@
# This file is maintained automatically by "terraform init".
# Manual edits may be lost in future updates.
provider "registry.terraform.io/cloudflare/cloudflare" {
version = "4.11.0"
constraints = "4.11.0, ~> 4.11.0"
hashes = [
"h1:IumoPgFcYKiFQjEMU8IHAELBu9DVmFUHPFDOzralbJ4=",
"h1:vPD1Yuk9AgqRHRWC+x8mznnSaInoTwaVS5dgnseMz88=",
"zh:09d620903d0f191ab7dee88ce75833307a03c7a9f88dfb2c2a58025283b80ff4",
"zh:0fb59cccc066c867750d633d6dfea8b99e75f5545ae4e7c090be465c6858eb73",
"zh:16b35bf2b88a629c05aefc6ebdbcc039447ee23a5b32594d844ca83f92ac8507",
"zh:5cc3f5df54891bb9efab51cca3266c59a82fd7dcc5667aa3451562325002235a",
"zh:6f384c9ba3e844b41c3de8455a3b91e3e3b32c1fa34b8b1ece4eae36d347c67e",
"zh:8000b3567ba7a43837bb8ccf7fdbcd03cc30103ec6abed84a40ee1c5b99f933f",
"zh:8687603e979a5fe82f2a65bc0cfb2a20acce4d871b01f04ffeabb9aa17c079ca",
"zh:88ed3e07913ad564ae3ae3280c868054d85e37b16db250b9cbdfca0c58f75dce",
"zh:890df766e9b839623b1f0437355032a3c006226a6c200cd911e15ee1a9014e9f",
"zh:a1faa7112d35aee74eb2b90543570ea56209112c0e2c1c06ad503a9c2464676d",
"zh:a433640c433f1815ca3cf92927a3764669095b8c668a73363ca9017a0b1d0349",
"zh:a63b6cf55baaa37cd4bf98bce94b7624bb54efe5abf8b86f24384df7996229f0",
"zh:a6696b0bdadb17d6f2ef7702b922c4006b21b4125530b0a8ac3bcfce1aafe2d8",
"zh:b2b3e16aa9c9d10409132fa7f181598bb67a1e5684c54535745ce0e3dcbd5d23",
"zh:d8c65b2e8a18141bb3ee53c7bf37422ff3679a67733702a631696586666ca885",
]
}
provider "registry.terraform.io/gavinbunney/kubectl" {
version = "1.14.0"
constraints = ">= 1.14.0, 1.14.0"
hashes = [
"h1:ItrWfCZMzM2JmvDncihBMalNLutsAk7kyyxVRaipftY=",
"h1:gLFn+RvP37sVzp9qnFCwngRjjFV649r6apjxvJ1E/SE=",
"zh:0350f3122ff711984bbc36f6093c1fe19043173fad5a904bce27f86afe3cc858",
"zh:07ca36c7aa7533e8325b38232c77c04d6ef1081cb0bac9d56e8ccd51f12f2030",
"zh:0c351afd91d9e994a71fe64bbd1662d0024006b3493bb61d46c23ea3e42a7cf5",
"zh:39f1a0aa1d589a7e815b62b5aa11041040903b061672c4cfc7de38622866cbc4",
"zh:428d3a321043b78e23c91a8d641f2d08d6b97f74c195c654f04d2c455e017de5",
"zh:4baf5b1de2dfe9968cc0f57fd4be5a741deb5b34ee0989519267697af5f3eee5",
"zh:6131a927f9dffa014ab5ca5364ac965fe9b19830d2bbf916a5b2865b956fdfcf",
"zh:c62e0c9fd052cbf68c5c2612af4f6408c61c7e37b615dc347918d2442dd05e93",
"zh:f0beffd7ce78f49ead612e4b1aefb7cb6a461d040428f514f4f9cc4e5698ac65",
]
}
provider "registry.terraform.io/hashicorp/archive" {
version = "2.4.0"
hashes = [
"h1:EtN1lnoHoov3rASpgGmh6zZ/W6aRCTgKC7iMwvFY1yc=",
"h1:cJokkjeH1jfpG4QEHdRx0t2j8rr52H33A7C/oX73Ok4=",
"zh:18e408596dd53048f7fc8229098d0e3ad940b92036a24287eff63e2caec72594",
"zh:392d4216ecd1a1fd933d23f4486b642a8480f934c13e2cae3c13b6b6a7e34a7b",
"zh:655dd1fa5ca753a4ace21d0de3792d96fff429445717f2ce31c125d19c38f3ff",
"zh:70dae36c176aa2b258331ad366a471176417a94dd3b4985a911b8be9ff842b00",
"zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
"zh:7d8c8e3925f1e21daf73f85983894fbe8868e326910e6df3720265bc657b9c9c",
"zh:a032ec0f0aee27a789726e348e8ad20778c3a1c9190ef25e7cff602c8d175f44",
"zh:b8e50de62ba185745b0fe9713755079ad0e9f7ac8638d204de6762cc36870410",
"zh:c8ad0c7697a3d444df21ff97f3473a8604c8639be64afe3f31b8ec7ad7571e18",
"zh:df736c5a2a7c3a82c5493665f659437a22f0baf8c2d157e45f4dd7ca40e739fc",
"zh:e8ffbf578a0977074f6d08aa8734e36c726e53dc79894cfc4f25fadc4f45f1df",
"zh:efea57ff23b141551f92b2699024d356c7ffd1a4ad62931da7ed7a386aef7f1f",
]
}
provider "registry.terraform.io/hashicorp/aws" {
version = "5.10.0"
constraints = ">= 3.72.0, >= 4.3.0, >= 4.8.0, >= 4.9.0, >= 4.10.0, >= 4.13.0, >= 4.30.0, >= 4.47.0, >= 4.67.0, >= 5.0.0, ~> 5.10.0"
hashes = [
"h1:AgF54/79Nb/oQjbAMMewENSIa1PEScMn20Xa91hZR2g=",
"h1:ll2mC5mMF+Tm/+tmDQ6p6h3oAFpMSbZsA54STMZegwI=",
"zh:24f8b40ba25521ec809906623ce1387542f3da848952167bc960663583a7b2c7",
"zh:3c12afbda4e8ed44ab8315d16bbba4329ef3f18ffe3c0d5ea456dd05472fa610",
"zh:4da2de97535c7fb51ede8ef9b6bd45c790005aec36daac4317a6175d2ff632fd",
"zh:5631fd3c02c5abe5e51a73bd77ddeaaf97b2d508845ea03bc1e5955b52d94706",
"zh:5bdef27b4e5b2dcd0661125fcc1e70826d545903b1e19bb8d28d2a0c812468d5",
"zh:7b7f6b3e00ad4b7bfaa9872388f7b8014d8c9a1fe5c3f9f57865535865727633",
"zh:935f7a599a3f55f69052b096491262d59787625ce5d52f729080328e5088e823",
"zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425",
"zh:a451a24f6675f8ad643a9b218cdb54c2af75a53d6a712daff46f64b81ec61032",
"zh:a5bcf820baefdc9f455222878f276a7f406a1092ac7b4c0cdbd6e588bff84847",
"zh:c9ab7b838a75bbcacc298658c1a04d1f0ee5935a928d821afcbe08c98cca7c5f",
"zh:d83855b6d66aaa03b1e66e03b7d0a4d1c9f992fce06f00011edde2a6ad6d91d6",
"zh:f1793e9a1e3ced98ca301ef1a294f46c06f77f6eb10f4d67ffef87ea60835421",
"zh:f366c99ddb16d75e07a687a60c015e8e2e0cdb593dea902385629571bd604859",
"zh:fb3ec60ea72144f480f495634c6d3e7a7638d7061a77c228a30768c1ae0b91f6",
]
}
provider "registry.terraform.io/hashicorp/cloudinit" {
version = "2.3.2"
constraints = ">= 2.0.0"
hashes = [
"h1:Vl0aixAYTV/bjathX7VArC5TVNkxBCsi3Vq7R4z1uvc=",
"h1:ocyv0lvfyvzW4krenxV5CL4Jq5DiA3EUfoy8DR6zFMw=",
"zh:2487e498736ed90f53de8f66fe2b8c05665b9f8ff1506f751c5ee227c7f457d1",
"zh:3d8627d142942336cf65eea6eb6403692f47e9072ff3fa11c3f774a3b93130b3",
"zh:434b643054aeafb5df28d5529b72acc20c6f5ded24decad73b98657af2b53f4f",
"zh:436aa6c2b07d82aa6a9dd746a3e3a627f72787c27c80552ceda6dc52d01f4b6f",
"zh:458274c5aabe65ef4dbd61d43ce759287788e35a2da004e796373f88edcaa422",
"zh:54bc70fa6fb7da33292ae4d9ceef5398d637c7373e729ed4fce59bd7b8d67372",
"zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
"zh:893ba267e18749c1a956b69be569f0d7bc043a49c3a0eb4d0d09a8e8b2ca3136",
"zh:95493b7517bce116f75cdd4c63b7c82a9d0d48ec2ef2f5eb836d262ef96d0aa7",
"zh:9ae21ab393be52e3e84e5cce0ef20e690d21f6c10ade7d9d9d22b39851bfeddc",
"zh:cc3b01ac2472e6d59358d54d5e4945032efbc8008739a6d4946ca1b621a16040",
"zh:f23bfe9758f06a1ec10ea3a81c9deedf3a7b42963568997d84a5153f35c5839a",
]
}
provider "registry.terraform.io/hashicorp/external" {
version = "2.3.1"
constraints = ">= 1.0.0"
hashes = [
"h1:bROCw6g5D/3fFnWeJ01L4IrdnJl1ILU8DGDgXCtYzaY=",
"h1:gznGscVJ0USxy4CdihpjRKPsKvyGr/zqPvBoFLJTQDc=",
"zh:001e2886dc81fc98cf17cf34c0d53cb2dae1e869464792576e11b0f34ee92f54",
"zh:2eeac58dd75b1abdf91945ac4284c9ccb2bfb17fa9bdb5f5d408148ff553b3ee",
"zh:2fc39079ba61411a737df2908942e6970cb67ed2f4fb19090cd44ce2082903dd",
"zh:472a71c624952cff7aa98a7b967f6c7bb53153dbd2b8f356ceb286e6743bb4e2",
"zh:4cff06d31272aac8bc35e9b7faec42cf4554cbcbae1092eaab6ab7f643c215d9",
"zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
"zh:7ed16ccd2049fa089616b98c0bd57219f407958f318f3c697843e2397ddf70df",
"zh:842696362c92bf2645eb85c739410fd51376be6c488733efae44f4ce688da50e",
"zh:8985129f2eccfd7f1841ce06f3bf2bbede6352ec9e9f926fbaa6b1a05313b326",
"zh:a5f0602d8ec991a5411ef42f872aa90f6347e93886ce67905c53cfea37278e05",
"zh:bf4ab82cbe5256dcef16949973bf6aa1a98c2c73a98d6a44ee7bc40809d002b8",
"zh:e70770be62aa70198fa899526d671643ff99eecf265bf1a50e798fc3480bd417",
]
}
provider "registry.terraform.io/hashicorp/helm" {
version = "2.10.1"
constraints = ">= 2.4.1, >= 2.5.1"
hashes = [
"h1:ctDhNJU4tEcyoUgPzwKuJmbDIqUl25mCY+s/lVHP6Sg=",
"h1:rssAXPIBWhumMtToGhh63w1euKOgVOi7+9LK6qZtDUQ=",
"zh:0717312baed39fb0a00576297241b69b419880cad8771bf72dec97ebdc96b200",
"zh:0e0e287b4e8429a0700143c8159764502eba0b33b1d094bf0d4ef4d93c7802cb",
"zh:4f74605377dab4065aaad35a2c5fa6186558c6e2e57b9058bdc8a62cf91857b9",
"zh:505f4af4dedb7a4f8f45b4201900b8e16216bdc2a01cc84fe13cdbf937570e7e",
"zh:83f37fe692513c0ce307d487248765383e00f9a84ed95f993ce0d3efdf4204d3",
"zh:840e5a84e1b5744f0211f611a2c6890da58016a40aafd5971f12285164d4e29b",
"zh:8c03d8dee292fa0367b0511cf3e95b706e034f78025f5dff0388116e1798bf47",
"zh:937800d1860f6b3adbb20e65f11e5fcd940b21ce8bdb48198630426244691325",
"zh:c1853aa5cbbdd1d46f4b169e84c3482103f0e8575a9bb044dbde908e27348c5d",
"zh:c9b0f640590da20931c30818b0b0587aa517d5606cb6e8052e4e4bf38f97b54d",
"zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c",
"zh:fe8bd4dd09dc7ca218959eda1ced9115408c2cdc9b4a76964bfa455f3bcadfd3",
]
}
provider "registry.terraform.io/hashicorp/kubernetes" {
version = "2.22.0"
constraints = ">= 2.6.1, >= 2.10.0"
hashes = [
"h1:DJr88+52tPK4Ft9xltF6YL+sRz8HWLP2ZOfFiKSB5Dc=",
"h1:b6Wj111/wsMNg8FrHFXrf4mCZFtSXKHx4JvbZh3YTCY=",
"zh:1eac662b1f238042b2068401e510f0624efaf51fd6a4dd9c49d710a49d383b61",
"zh:4c35651603493437b0b13e070148a330c034ac62c8967c2de9da6620b26adca4",
"zh:50c0e8654efb46e3a3666c638ca2e0c8aec07f985fbc80f9205bed960386dc9b",
"zh:5f65194ddd6ea7e89b378297d882083a4b84962edb35dd35752f0c7e9d6282a0",
"zh:6fc0c2d65864324edde4db84f528268065df58229fc3ee321626687b0e603637",
"zh:73c58d007aba7f67c0aa9029794e10c2517bec565b7cb57d0f5948ea3f30e407",
"zh:7d6fc9d3c1843baccd2e1fc56317925a2f9df372427d30fcb5052d123adc887a",
"zh:a0ad9eb863b51586ea306c5f2beef74476c96684aed41a3ee99eb4b6d8898d01",
"zh:e218fcfbf4994ff741408a023a9d9eb6c697ce9f63ce5540d3b35226d86c963e",
"zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c",
"zh:f95625f317795f0e38cc6293dd31c85863f4e225209d07d1e233c50d9295083c",
"zh:f96e0923a632bc430267fe915794972be873887f5e761ed11451d67202e256c8",
]
}
provider "registry.terraform.io/hashicorp/local" {
version = "2.4.0"
constraints = ">= 1.0.0, >= 2.1.0"
hashes = [
"h1:R97FTYETo88sT2VHfMgkPU3lzCsZLunPftjSI5vfKe8=",
"h1:ZUEYUmm2t4vxwzxy1BvN1wL6SDWrDxfH7pxtzX8c6d0=",
"zh:53604cd29cb92538668fe09565c739358dc53ca56f9f11312b9d7de81e48fab9",
"zh:66a46e9c508716a1c98efbf793092f03d50049fa4a83cd6b2251e9a06aca2acf",
"zh:70a6f6a852dd83768d0778ce9817d81d4b3f073fab8fa570bff92dcb0824f732",
"zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
"zh:82a803f2f484c8b766e2e9c32343e9c89b91997b9f8d2697f9f3837f62926b35",
"zh:9708a4e40d6cc4b8afd1352e5186e6e1502f6ae599867c120967aebe9d90ed04",
"zh:973f65ce0d67c585f4ec250c1e634c9b22d9c4288b484ee2a871d7fa1e317406",
"zh:c8fa0f98f9316e4cfef082aa9b785ba16e36ff754d6aba8b456dab9500e671c6",
"zh:cfa5342a5f5188b20db246c73ac823918c189468e1382cb3c48a9c0c08fc5bf7",
"zh:e0e2b477c7e899c63b06b38cd8684a893d834d6d0b5e9b033cedc06dd7ffe9e2",
"zh:f62d7d05ea1ee566f732505200ab38d94315a4add27947a60afa29860822d3fc",
"zh:fa7ce69dde358e172bd719014ad637634bbdabc49363104f4fca759b4b73f2ce",
]
}
provider "registry.terraform.io/hashicorp/null" {
version = "3.2.1"
constraints = ">= 2.0.0, >= 3.0.0, >= 3.1.0"
hashes = [
"h1:FbGfc+muBsC17Ohy5g806iuI1hQc4SIexpYCrQHQd8w=",
"h1:ydA0/SNRVB1o95btfshvYsmxA+jZFRZcvKzZSB+4S1M=",
"zh:58ed64389620cc7b82f01332e27723856422820cfd302e304b5f6c3436fb9840",
"zh:62a5cc82c3b2ddef7ef3a6f2fedb7b9b3deff4ab7b414938b08e51d6e8be87cb",
"zh:63cff4de03af983175a7e37e52d4bd89d990be256b16b5c7f919aff5ad485aa5",
"zh:74cb22c6700e48486b7cabefa10b33b801dfcab56f1a6ac9b6624531f3d36ea3",
"zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
"zh:79e553aff77f1cfa9012a2218b8238dd672ea5e1b2924775ac9ac24d2a75c238",
"zh:a1e06ddda0b5ac48f7e7c7d59e1ab5a4073bbcf876c73c0299e4610ed53859dc",
"zh:c37a97090f1a82222925d45d84483b2aa702ef7ab66532af6cbcfb567818b970",
"zh:e4453fbebf90c53ca3323a92e7ca0f9961427d2f0ce0d2b65523cc04d5d999c2",
"zh:e80a746921946d8b6761e77305b752ad188da60688cfd2059322875d363be5f5",
"zh:fbdb892d9822ed0e4cb60f2fedbdbb556e4da0d88d3b942ae963ed6ff091e48f",
"zh:fca01a623d90d0cad0843102f9b8b9fe0d3ff8244593bd817f126582b52dd694",
]
}
provider "registry.terraform.io/hashicorp/random" {
version = "3.5.1"
constraints = ">= 2.2.0, >= 3.0.0, ~> 3.5.1"
hashes = [
"h1:IL9mSatmwov+e0+++YX2V6uel+dV6bn+fC/cnGDK3Ck=",
"h1:VSnd9ZIPyfKHOObuQCaKfnjIHRtR7qTw19Rz8tJxm+k=",
"zh:04e3fbd610cb52c1017d282531364b9c53ef72b6bc533acb2a90671957324a64",
"zh:119197103301ebaf7efb91df8f0b6e0dd31e6ff943d231af35ee1831c599188d",
"zh:4d2b219d09abf3b1bb4df93d399ed156cadd61f44ad3baf5cf2954df2fba0831",
"zh:6130bdde527587bbe2dcaa7150363e96dbc5250ea20154176d82bc69df5d4ce3",
"zh:6cc326cd4000f724d3086ee05587e7710f032f94fc9af35e96a386a1c6f2214f",
"zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
"zh:b6d88e1d28cf2dfa24e9fdcc3efc77adcdc1c3c3b5c7ce503a423efbdd6de57b",
"zh:ba74c592622ecbcef9dc2a4d81ed321c4e44cddf7da799faa324da9bf52a22b2",
"zh:c7c5cde98fe4ef1143bd1b3ec5dc04baf0d4cc3ca2c5c7d40d17c0e9b2076865",
"zh:dac4bad52c940cd0dfc27893507c1e92393846b024c5a9db159a93c534a3da03",
"zh:de8febe2a2acd9ac454b844a4106ed295ae9520ef54dc8ed2faf29f12716b602",
"zh:eab0d0495e7e711cca367f7d4df6e322e6c562fc52151ec931176115b83ed014",
]
}
provider "registry.terraform.io/hashicorp/time" {
version = "0.9.1"
constraints = ">= 0.7.0, >= 0.8.0"
hashes = [
"h1:NUv/YtEytDQncBQ2mTxnUZEy/rmDlPYmE9h2iokR0vk=",
"h1:VxyoYYOCaJGDmLz4TruZQTSfQhvwEcMxvcKclWdnpbs=",
"zh:00a1476ecf18c735cc08e27bfa835c33f8ac8fa6fa746b01cd3bcbad8ca84f7f",
"zh:3007f8fc4a4f8614c43e8ef1d4b0c773a5de1dcac50e701d8abc9fdc8fcb6bf5",
"zh:5f79d0730fdec8cb148b277de3f00485eff3e9cf1ff47fb715b1c969e5bbd9d4",
"zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
"zh:8c8094689a2bed4bb597d24a418bbbf846e15507f08be447d0a5acea67c2265a",
"zh:a6d9206e95d5681229429b406bc7a9ba4b2d9b67470bda7df88fa161508ace57",
"zh:aa299ec058f23ebe68976c7581017de50da6204883950de228ed9246f309e7f1",
"zh:b129f00f45fba1991db0aa954a6ba48d90f64a738629119bfb8e9a844b66e80b",
"zh:ef6cecf5f50cda971c1b215847938ced4cb4a30a18095509c068643b14030b00",
"zh:f1f46a4f6c65886d2dd27b66d92632232adc64f92145bf8403fe64d5ffa5caea",
"zh:f79d6155cda7d559c60d74883a24879a01c4d5f6fd7e8d1e3250f3cd215fb904",
"zh:fd59fa73074805c3575f08cd627eef7acda14ab6dac2c135a66e7a38d262201c",
]
}
provider "registry.terraform.io/hashicorp/tls" {
version = "4.0.4"
constraints = ">= 3.0.0"
hashes = [
"h1:GZcFizg5ZT2VrpwvxGBHQ/hO9r6g0vYdQqx3bFD3anY=",
"h1:pe9vq86dZZKCm+8k1RhzARwENslF3SXb9ErHbQfgjXU=",
"zh:23671ed83e1fcf79745534841e10291bbf34046b27d6e68a5d0aab77206f4a55",
"zh:45292421211ffd9e8e3eb3655677700e3c5047f71d8f7650d2ce30242335f848",
"zh:59fedb519f4433c0fdb1d58b27c210b27415fddd0cd73c5312530b4309c088be",
"zh:5a8eec2409a9ff7cd0758a9d818c74bcba92a240e6c5e54b99df68fff312bbd5",
"zh:5e6a4b39f3171f53292ab88058a59e64825f2b842760a4869e64dc1dc093d1fe",
"zh:810547d0bf9311d21c81cc306126d3547e7bd3f194fc295836acf164b9f8424e",
"zh:824a5f3617624243bed0259d7dd37d76017097dc3193dac669be342b90b2ab48",
"zh:9361ccc7048be5dcbc2fafe2d8216939765b3160bd52734f7a9fd917a39ecbd8",
"zh:aa02ea625aaf672e649296bce7580f62d724268189fe9ad7c1b36bb0fa12fa60",
"zh:c71b4cd40d6ec7815dfeefd57d88bc592c0c42f5e5858dcc88245d371b4b8b1e",
"zh:dabcd52f36b43d250a3d71ad7abfa07b5622c69068d989e60b79b2bb4f220316",
"zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c",
]
}
provider "registry.terraform.io/kreuzwerker/docker" {
version = "2.16.0"
constraints = "~> 2.16.0"
hashes = [
"h1:OcTn2QyCQNjDiJYy1vqQFmz2dxJdOF/2/HBXBvGxU2E=",
"h1:aslxshC6HTeDoZuygVzqDmyFCbCizZs7AWHDWk1p/6c=",
"zh:0ff8aa7884c6dae90e6f245bb9d37898735f89e095ba53413f2f364db4d11a77",
"zh:4101f4c909477f3a8225829b7063e5c5a2e2986a6163e0f113af040b5feab61f",
"zh:59db110d2b6c620cc12a1741d81ed8d1dd7fb0540024428fefbb57e8bebe5b60",
"zh:6e134983f195ea0273ac042f0a2df14158d676a24e8dd140ca0357f3efc3fd61",
"zh:7de1de3cc1eacb2ef2693207f5c5f54fa4814ae8c024b8b3c2a0923c82fd6f14",
"zh:a6659fbc7c45fbb60c7c9bf06724eb6084711f1b79c720ef8512a4367e63cbe5",
"zh:ae97c721431517d8c71f8cede91d734d2f2372a1bfef0c3bba43b54c0f8b1cee",
"zh:b3cbd47d5f0cb522b6dd3561ccd2f491fb6afb577372718e0663d12cfeef30e9",
"zh:b64af7c6ad8870c11677874f6cd13322aa03d2190391a120be17304ca324ea1c",
"zh:c363747bae968af997eaf22193168451523e92b59aee8aee135d3b27db132366",
"zh:c40721250642157b2a72d8db44fa09de0f7635ba4b0e2ebf5527570f3988e62f",
"zh:e97707609e346bf463d539099faa8790f2f453cfbd0b880327b6eae16ca4f213",
"zh:f4a23ce27cb430f91895466b3e2d132c534fa2b58808f6771235d76e696f4972",
"zh:fd634e973eb2b6483a1ce9251801a393d04cb496f8e83ffcf3f0c4cad8c18f4c",
]
}
provider "registry.terraform.io/paultyng/git" {
version = "0.1.0"
constraints = "~> 0.1.0"
hashes = [
"h1:2BsazHD/QR4AvjGB4laTWU8VC9IFnMrSZ+2gCunQ1JI=",
"h1:nz3VfU3LHDUQFdILoXq8O0FWbQZfCmXhpQOTKRRzEaY=",
"zh:0d593ac990f711171875ba5fc838f0087df84ddb1c69154ee630def5984931ea",
"zh:3895c2719f42e93fc993474859b34de87d90e2c47dfb757d435b9b57945195e4",
"zh:3a90ce559a3589628a2d6820a9d76a354763c268b0c173982ff773e022032856",
"zh:42339a6084095e37d0c843907dcabe66989949ea3f0025f6f1f9d8583d7da779",
"zh:435522beccaedf89bc39eed495393194b43156d1730ef45c29faa584552dc355",
"zh:87b4ee4f521283daaa0d63dd7949dc59f700b92e246e4aeb06510c01842a3c8b",
"zh:997aca77ddc1411dd601ea1fa2e455be9531c3e3c0f0917e8f2423ffd4ffb9ba",
"zh:a70e98ce6ef7a8256286ab791bc231777b76c8f038da4b9eccf399d2b22051fb",
"zh:af9301520e8befe3ec6d1125e10cc0724b318590f5680f12032c8bdc3b0c827d",
"zh:d995a3b8eaa5ac61744d49127fbf68b4c32e16d3c67d570edda2af26113b92a5",
"zh:e8b5c7354a02c54efc026d8289ce9d3784f58abd673a78e80bd4fb073dd75101",
]
}
provider "registry.terraform.io/terraform-aws-modules/http" {
version = "2.4.1"
constraints = "2.4.1"
hashes = [
"h1:ZnkXcawrIr611RvZpoDzbtPU7SVFyHym+7p1t+PQh20=",
"h1:fHqAXle/P/fT2k+HEyTqYVE+/RvpQAaBr6xXZgM66es=",
"zh:0111f54de2a9815ded291f23136d41f3d2731c58ea663a2e8f0fef02d377d697",
"zh:0740152d76f0ccf54f4d0e8e0753739a5233b022acd60b5d2353d248c4c17204",
"zh:569518f46809ec9cdc082b4dfd4e828236eee2b50f87b301d624cfd83b8f5b0d",
"zh:7669f7691de91eec9f381e9a4be81aa4560f050348a86c6ea7804925752a01bb",
"zh:81cd53e796ec806aca2d8e92a2aed9135661e170eeff6cf0418e54f98816cd05",
"zh:82f01abd905090f978b169ac85d7a5952322a5f0f460269dd981b3596652d304",
"zh:9a235610066e0f7e567e69c23a53327271a6fc568b06bf152d8fe6594749ed2b",
"zh:aeabdd8e633d143feb67c52248c85358951321e35b43943aeab577c005abd30a",
"zh:c20d22dba5c79731918e7192bc3d0b364d47e98a74f47d287e6cc66236bc0ed0",
"zh:c4fea2cb18c31ed7723deec5ebaff85d6795bb6b6ed3b954794af064d17a7f9f",
"zh:e21e88b6e7e55b9f29b046730d9928c65a4f181fd5f60a42f1cd41b46a0a938d",
"zh:eddb888a74dea348a0acdfee13a08875bacddde384bd9c28342a534269665568",
"zh:f46d5f1403b8d8dfafab9bdd7129d3080bb62a91ea726f477fd43560887b8c4a",
]
}

1
infrastructure/sandbox/Data/.gitignore vendored
View file

@ -1 +0,0 @@
lambda.zip

5
infrastructure/sandbox/Data/README.md
View file

@ -1,5 +0,0 @@
# The data pipeline
The data pipeline takes data from S3 using S3 notifications,
filters for only the successful requests, then enriches the data with geoip data,
then pipes it to kinesis. From kinesis, we stream the data to an Elasticsearch cluster for now,
but this design allows for expansion into Salesforce and Mixpanel later on.

BIN
infrastructure/sandbox/Data/lambda/__pycache__/geohash.cpython-310.pyc
View file

Binary file not shown.

BIN
infrastructure/sandbox/Data/lambda/__pycache__/jpgrid.cpython-310.pyc
View file

Binary file not shown.

BIN
infrastructure/sandbox/Data/lambda/__pycache__/jpiarea.cpython-310.pyc
View file

Binary file not shown.

BIN
infrastructure/sandbox/Data/lambda/__pycache__/quadtree.cpython-310.pyc
View file

Binary file not shown.

BIN
infrastructure/sandbox/Data/lambda/_geohash.cpython-310-x86_64-linux-gnu.so
View file

Binary file not shown.

BIN
infrastructure/sandbox/Data/lambda/_geoip_geolite2/GeoLite2-City.mmdb
View file

Binary file not shown.
Side by side

Before

Width:  |  Height:  |  Size: 33 MiB

9
infrastructure/sandbox/Data/lambda/_geoip_geolite2/__init__.py
View file

@ -1,9 +0,0 @@
import os
database_name = 'GeoLite2-City.mmdb'
def loader(database, mod):
filename = os.path.join(os.path.dirname(__file__), database_name)
return mod.open_database(filename)

BIN
infrastructure/sandbox/Data/lambda/_geoip_geolite2/__pycache__/__init__.cpython-310.pyc
View file

Binary file not shown.

2
infrastructure/sandbox/Data/lambda/elasticsearch-6.8.2.dist-info/AUTHORS
View file

@ -1,2 +0,0 @@
For a list of all our amazing authors please see the contributors page:
https://github.com/elastic/elasticsearch-py/graphs/contributors

1
infrastructure/sandbox/Data/lambda/elasticsearch-6.8.2.dist-info/INSTALLER
View file

@ -1 +0,0 @@
pip

176
infrastructure/sandbox/Data/lambda/elasticsearch-6.8.2.dist-info/LICENSE
View file

@ -1,176 +0,0 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.

193
infrastructure/sandbox/Data/lambda/elasticsearch-6.8.2.dist-info/METADATA
View file

@ -1,193 +0,0 @@
Metadata-Version: 2.1
Name: elasticsearch
Version: 6.8.2
Summary: Python client for Elasticsearch
Home-page: https://github.com/elastic/elasticsearch-py
Author: Honza Král, Nick Lang
Author-email: honza.kral@gmail.com, nick@nicklang.com
Maintainer: Seth Michael Larson
Maintainer-email: seth.larson@elastic.co
License: Apache-2.0
Platform: UNKNOWN
Classifier: Development Status :: 5 - Production/Stable
Classifier: License :: OSI Approved :: Apache Software License
Classifier: Intended Audience :: Developers
Classifier: Operating System :: OS Independent
Classifier: Programming Language :: Python
Classifier: Programming Language :: Python :: 2
Classifier: Programming Language :: Python :: 2.6
Classifier: Programming Language :: Python :: 2.7
Classifier: Programming Language :: Python :: 3
Classifier: Programming Language :: Python :: 3.2
Classifier: Programming Language :: Python :: 3.3
Classifier: Programming Language :: Python :: 3.4
Classifier: Programming Language :: Python :: 3.5
Classifier: Programming Language :: Python :: 3.6
Classifier: Programming Language :: Python :: Implementation :: CPython
Classifier: Programming Language :: Python :: Implementation :: PyPy
Requires-Python: >=2.6, !=3.0.*, !=3.1.*, !=3.2.*, <4
Description-Content-Type: text/x-rst
Requires-Dist: urllib3 (>=1.21.1)
Provides-Extra: develop
Requires-Dist: requests (<3.0.0,>=2.0.0) ; extra == 'develop'
Requires-Dist: nose ; extra == 'develop'
Requires-Dist: coverage ; extra == 'develop'
Requires-Dist: mock ; extra == 'develop'
Requires-Dist: pyyaml ; extra == 'develop'
Requires-Dist: nosexcover ; extra == 'develop'
Requires-Dist: numpy ; extra == 'develop'
Requires-Dist: pandas ; extra == 'develop'
Requires-Dist: sphinx (<1.7) ; extra == 'develop'
Requires-Dist: sphinx-rtd-theme ; extra == 'develop'
Provides-Extra: requests
Requires-Dist: requests (<3.0.0,>=2.4.0) ; extra == 'requests'
Python Elasticsearch Client
===========================
Official low-level client for Elasticsearch. Its goal is to provide common
ground for all Elasticsearch-related code in Python; because of this it tries
to be opinion-free and very extendable.
For a more high level client library with more limited scope, have a look at
`elasticsearch-dsl`_ - a more pythonic library sitting on top of
``elasticsearch-py``.
It provides a more convenient and idiomatic way to write and manipulate
`queries`_. It stays close to the Elasticsearch JSON DSL, mirroring its
terminology and structure while exposing the whole range of the DSL from Python
either directly using defined classes or a queryset-like expressions.
It also provides an optional `persistence layer`_ for working with documents as
Python objects in an ORM-like fashion: defining mappings, retrieving and saving
documents, wrapping the document data in user-defined classes.
.. _elasticsearch-dsl: https://elasticsearch-dsl.readthedocs.io/
.. _queries: https://elasticsearch-dsl.readthedocs.io/en/latest/search_dsl.html
.. _persistence layer: https://elasticsearch-dsl.readthedocs.io/en/latest/persistence.html#doctype
Compatibility
-------------
The library is compatible with all Elasticsearch versions since ``0.90.x`` but you
**have to use a matching major version**:
For **Elasticsearch 6.0** and later, use the major version 6 (``6.x.y``) of the
library.
For **Elasticsearch 5.0** and later, use the major version 5 (``5.x.y``) of the
library.
For **Elasticsearch 2.0** and later, use the major version 2 (``2.x.y``) of the
library, and so on.
The recommended way to set your requirements in your `setup.py` or
`requirements.txt` is::
# Elasticsearch 6.x
elasticsearch>=6.0.0,<7.0.0
# Elasticsearch 5.x
elasticsearch>=5.0.0,<6.0.0
# Elasticsearch 2.x
elasticsearch>=2.0.0,<3.0.0
If you have a need to have multiple versions installed at the same time older
versions are also released as ``elasticsearch2``, ``elasticsearch5`` and ``elasticsearch6``.
Installation
------------
Install the ``elasticsearch`` package for Elasticsearch 6.x with `pip
<https://pypi.python.org/pypi/elasticsearch>`_::
pip install "elasticsearch>=6,<7"
Example use
-----------
Simple use-case::
>>> from datetime import datetime
>>> from elasticsearch import Elasticsearch
# by default we connect to localhost:9200
>>> es = Elasticsearch()
# create an index in elasticsearch, ignore status code 400 (index already exists)
>>> es.indices.create(index='my-index', ignore=400)
{u'acknowledged': True}
# datetimes will be serialized
>>> es.index(index="my-index", doc_type="test-type", id=42, body={"any": "data", "timestamp": datetime.now()})
{u'_id': u'42', u'_index': u'my-index', u'_type': u'test-type', u'_version': 1, u'ok': True}
# but not deserialized
>>> es.get(index="my-index", doc_type="test-type", id=42)['_source']
{u'any': u'data', u'timestamp': u'2013-05-12T19:45:31.804229'}
`Full documentation`_.
.. _Full documentation: https://elasticsearch-py.readthedocs.io/
Elastic Cloud (and SSL) use-case::
>>> from elasticsearch import Elasticsearch
>>> es = Elasticsearch("https://elasticsearch.url:port", http_auth=('elastic','yourpassword'))
>>> es.info()
Using SSL Context with a self-signed cert use-case::
>>> from elasticsearch import Elasticsearch
>>> from ssl import create_default_context
>>> context = create_default_context(cafile="path/to/cafile.pem")
>>> es = Elasticsearch("https://elasticsearch.url:port", ssl_context=context, http_auth=('elastic','yourpassword'))
>>> es.info()
Features
--------
The client's features include:
* translating basic Python data types to and from json (datetimes are not
decoded for performance reasons)
* configurable automatic discovery of cluster nodes
* persistent connections
* load balancing (with pluggable selection strategy) across all available nodes
* failed connection penalization (time based - failed connections won't be
retried until a timeout is reached)
* support for ssl and http authentication
* thread safety
* pluggable architecture
License
-------
Copyright 2017 Elasticsearch
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
Build status
------------
.. image:: https://readthedocs.org/projects/elasticsearch-py/badge/?version=latest&style=flat
:target: https://elasticsearch-py.readthedocs.io/en/master/
.. image:: https://clients-ci.elastic.co/job/elastic+elasticsearch-py+master/badge/icon
:target: https://clients-ci.elastic.co/job/elastic+elasticsearch-py+master/

78
infrastructure/sandbox/Data/lambda/elasticsearch-6.8.2.dist-info/RECORD
View file

@ -1,78 +0,0 @@
elasticsearch-6.8.2.dist-info/AUTHORS,sha256=lzWXD7E6TlSJkJAHfegykZOG8PQ3rEsm-xRT6uysmDg,136
elasticsearch-6.8.2.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
elasticsearch-6.8.2.dist-info/LICENSE,sha256=XfKg2H1sVi8OoRxoisUlMqoo10TKvHmU_wU39ks7MyA,10143
elasticsearch-6.8.2.dist-info/METADATA,sha256=5qN-GB11HL43f83oiTMP6tMvJByEQITBSO8Z1nPUX1U,7028
elasticsearch-6.8.2.dist-info/RECORD,,
elasticsearch-6.8.2.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
elasticsearch-6.8.2.dist-info/WHEEL,sha256=Z-nyYpwrcSqxfdux5Mbn_DQ525iP7J2DG3JgGvOYyTQ,110
elasticsearch-6.8.2.dist-info/top_level.txt,sha256=Jp2bLWq49skvCN4YCZsg1Hfn_NDLgleC-x-Bn01_HgM,14
elasticsearch/__init__.py,sha256=T0k9G4d0hHW7tzffNGXG6uFtH2cQUW8_1sTQh0fyAp8,1511
elasticsearch/__pycache__/__init__.cpython-310.pyc,,
elasticsearch/__pycache__/compat.cpython-310.pyc,,
elasticsearch/__pycache__/connection_pool.cpython-310.pyc,,
elasticsearch/__pycache__/exceptions.cpython-310.pyc,,
elasticsearch/__pycache__/serializer.cpython-310.pyc,,
elasticsearch/__pycache__/transport.cpython-310.pyc,,
elasticsearch/__pycache__/utils.cpython-310.pyc,,
elasticsearch/client/__init__.py,sha256=zR2WSLCA7T4Rx3asB4xmKzkwRzzxnmwiOlLN-EN2aIs,86963
elasticsearch/client/__pycache__/__init__.cpython-310.pyc,,
elasticsearch/client/__pycache__/cat.cpython-310.pyc,,
elasticsearch/client/__pycache__/cluster.cpython-310.pyc,,
elasticsearch/client/__pycache__/indices.cpython-310.pyc,,
elasticsearch/client/__pycache__/ingest.cpython-310.pyc,,
elasticsearch/client/__pycache__/nodes.cpython-310.pyc,,
elasticsearch/client/__pycache__/remote.cpython-310.pyc,,
elasticsearch/client/__pycache__/snapshot.cpython-310.pyc,,
elasticsearch/client/__pycache__/tasks.cpython-310.pyc,,
elasticsearch/client/__pycache__/utils.cpython-310.pyc,,
elasticsearch/client/cat.py,sha256=sptUQCjDN9gdrHXYYxgbyKFkYOdJAyUGx7t2L_QjwSc,22396
elasticsearch/client/cluster.py,sha256=6GRgj-4I4UJjtzHdUN9iuckD-orxb_nJ0aAsfWn6sKo,10187
elasticsearch/client/indices.py,sha256=f4EsC3TjSpK-BF24Zfk-Q3Vt5NeHyqrFfumYqXb22rg,53453
elasticsearch/client/ingest.py,sha256=R-LLnE3bJ5HUKwsrALxKuQBzbU8G9TmiAiPcIKZu5zg,3771
elasticsearch/client/nodes.py,sha256=2BxjaLxizaqj-zwwVP_LM4KbV89pDKcfZucnP4QZIlQ,7257
elasticsearch/client/remote.py,sha256=jnV51VIZx8O6m9FWLiPB9jaDtjhQlU4_qdq_xr7HDbI,1141
elasticsearch/client/snapshot.py,sha256=IBJEmn7pZ6bISP-ZeaTakr6YO9H15EQn_jBqS_Bdv7A,8911
elasticsearch/client/tasks.py,sha256=TP9gN-IQrp4F6--W2-Gg9y7EyP84koDr_UCPuUni3tA,3767
elasticsearch/client/utils.py,sha256=iunit4TkACv5Z4NYQI29O--DE4F2nPfe8HfY0mUVS7M,3731
elasticsearch/client/xpack/__init__.py,sha256=vXun8SJFQTIBZu2Kc44IUg8bMvJgY1O5myaoSfZUKTY,2596
elasticsearch/client/xpack/__pycache__/__init__.cpython-310.pyc,,
elasticsearch/client/xpack/__pycache__/deprecation.cpython-310.pyc,,
elasticsearch/client/xpack/__pycache__/graph.cpython-310.pyc,,
elasticsearch/client/xpack/__pycache__/license.cpython-310.pyc,,
elasticsearch/client/xpack/__pycache__/migration.cpython-310.pyc,,
elasticsearch/client/xpack/__pycache__/ml.cpython-310.pyc,,
elasticsearch/client/xpack/__pycache__/monitoring.cpython-310.pyc,,
elasticsearch/client/xpack/__pycache__/security.cpython-310.pyc,,
elasticsearch/client/xpack/__pycache__/watcher.cpython-310.pyc,,
elasticsearch/client/xpack/deprecation.py,sha256=au0BCNQ0UcuT9nSMMeMMhunUFYkd9NU0QTgXhaBTDBo,1286
elasticsearch/client/xpack/graph.py,sha256=fNsL7IaQNr6IptWvnImTmXmmltlCMN7YUNXR18f-jPc,1741
elasticsearch/client/xpack/license.py,sha256=T9NNuTtM8oRu9UesHsO9AfhkBUqO4NXQxRl4EQoUbIY,2006
elasticsearch/client/xpack/migration.py,sha256=AFEuUf3lbyVJuvNdPUMA0Po8F_OK55DLeG2A9xviEMM,2682
elasticsearch/client/xpack/ml.py,sha256=I-WJsVUE6zsBmPKxOVx0poWo1AZAzcM55UUUR1SlMUk,24716
elasticsearch/client/xpack/monitoring.py,sha256=8S9F8sjFIvSPYWfMbO7HbTFOsi3XhecWx_q3QXyvNkU,1906
elasticsearch/client/xpack/security.py,sha256=02s9qNH6Qd8OXZ1xBtAduzH4nuw2pcwO77TlCmr23xc,13016
elasticsearch/client/xpack/watcher.py,sha256=5XugO86Lr7FfeNBGLhhge-O0JXLXounRYE69yDkFlSg,6820
elasticsearch/compat.py,sha256=LYXKmjFhytJ50Lv2LYawuPT3QbBV85PpBzHKo1JXJIM,1342
elasticsearch/connection/__init__.py,sha256=niEqyjlEb5K6LYjppaLaT31ZkbjifvgTRc7nRfsA_jU,1053
elasticsearch/connection/__pycache__/__init__.cpython-310.pyc,,
elasticsearch/connection/__pycache__/base.cpython-310.pyc,,
elasticsearch/connection/__pycache__/http_requests.cpython-310.pyc,,
elasticsearch/connection/__pycache__/http_urllib3.cpython-310.pyc,,
elasticsearch/connection/__pycache__/pooling.cpython-310.pyc,,
elasticsearch/connection/base.py,sha256=y1SwwrgHikTUJ3wBgDTkgiF2dM-idJmQ6T90TU6Fx04,8942
elasticsearch/connection/http_requests.py,sha256=MML8JblofLw2QJa-S8VNwxyo-8r0k1SZWu69g_yCvd0,7089
elasticsearch/connection/http_urllib3.py,sha256=8yfIZyYVptMcmBpGkSGyHVUv_ZfRzUoa_eHp5eMlX9M,9583
elasticsearch/connection/pooling.py,sha256=i8f6R3nl8sC2CsYhsJzqc6EBQBN4msxM9Rdn8wKbCh8,1682
elasticsearch/connection_pool.py,sha256=HC_awZt0NUaII592PhgEvK53PYaxKJaJnYL8MCCAYgw,10847
elasticsearch/exceptions.py,sha256=fTDkJIEy_xfIpu4bMLdoi_KCWnlpkSUjmYOq92AfPcM,4357
elasticsearch/helpers/__init__.py,sha256=rRSQsrrCYJbDJXrvki8tSyiICAEyZwhwmTbGuu1XsUc,1204
elasticsearch/helpers/__pycache__/__init__.cpython-310.pyc,,
elasticsearch/helpers/__pycache__/actions.cpython-310.pyc,,
elasticsearch/helpers/__pycache__/errors.cpython-310.pyc,,
elasticsearch/helpers/__pycache__/test.cpython-310.pyc,,
elasticsearch/helpers/actions.py,sha256=MD4mvtpPKHywFA7EKEcC7zZ1JsjuO8qmARPLOy2UwTI,20700
elasticsearch/helpers/errors.py,sha256=rhPLN2qM8RS-HjGy5INss0zXUXFnKItKrY7BzTDnbCs,1200
elasticsearch/helpers/test.py,sha256=LSdTlhPS3naUH_wxMUG_AFl5VTR10ze7EjxgJmCsVaE,2641
elasticsearch/serializer.py,sha256=DqMwHr-7MJrpu9Lbq01RQ58CJvuR_hC-yYeFkAT6vc0,4372
elasticsearch/transport.py,sha256=DJ2INj9kSkX_MTCc-qrQ5Mule9CkDQ34KXwSINmjUCg,18089
elasticsearch/utils.py,sha256=6rY_mTQpfUfaixSi1QxPsSi-uKP_0z98ZE1kZlekL8g,1176

0
infrastructure/sandbox/Data/lambda/elasticsearch-6.8.2.dist-info/REQUESTED
View file

6
infrastructure/sandbox/Data/lambda/elasticsearch-6.8.2.dist-info/WHEEL
View file

@ -1,6 +0,0 @@
Wheel-Version: 1.0
Generator: bdist_wheel (0.36.2)
Root-Is-Purelib: true
Tag: py2-none-any
Tag: py3-none-any

1
infrastructure/sandbox/Data/lambda/elasticsearch-6.8.2.dist-info/top_level.txt
View file

@ -1 +0,0 @@
elasticsearch

46
infrastructure/sandbox/Data/lambda/elasticsearch/__init__.py
View file

@ -1,46 +0,0 @@
# Licensed to Elasticsearch B.V. under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch B.V. licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# flake8: noqa
from __future__ import absolute_import
VERSION = (6, 8, 2)
__version__ = VERSION
__versionstr__ = ".".join(map(str, VERSION))
import logging
try: # Python 2.7+
from logging import NullHandler
except ImportError:
class NullHandler(logging.Handler):
def emit(self, record):
pass
import sys
logger = logging.getLogger("elasticsearch")
logger.addHandler(logging.NullHandler())
from .client import Elasticsearch
from .transport import Transport
from .connection_pool import ConnectionPool, ConnectionSelector, RoundRobinSelector
from .serializer import JSONSerializer
from .connection import Connection, RequestsHttpConnection, Urllib3HttpConnection
from .exceptions import *

BIN
infrastructure/sandbox/Data/lambda/elasticsearch/__pycache__/__init__.cpython-310.pyc
View file

Binary file not shown.

BIN
infrastructure/sandbox/Data/lambda/elasticsearch/__pycache__/compat.cpython-310.pyc
View file

Binary file not shown.

BIN
infrastructure/sandbox/Data/lambda/elasticsearch/__pycache__/connection_pool.cpython-310.pyc
View file

Binary file not shown.

BIN
infrastructure/sandbox/Data/lambda/elasticsearch/__pycache__/exceptions.cpython-310.pyc
View file

Binary file not shown.

BIN
infrastructure/sandbox/Data/lambda/elasticsearch/__pycache__/serializer.cpython-310.pyc
View file

Binary file not shown.

BIN
infrastructure/sandbox/Data/lambda/elasticsearch/__pycache__/transport.cpython-310.pyc
View file

Binary file not shown.

BIN
infrastructure/sandbox/Data/lambda/elasticsearch/__pycache__/utils.cpython-310.pyc
View file

Binary file not shown.

1916
infrastructure/sandbox/Data/lambda/elasticsearch/client/__init__.py
View file

File diff suppressed because it is too large Load diff

BIN
infrastructure/sandbox/Data/lambda/elasticsearch/client/__pycache__/__init__.cpython-310.pyc
View file

Binary file not shown.

BIN
infrastructure/sandbox/Data/lambda/elasticsearch/client/__pycache__/cat.cpython-310.pyc
View file

Binary file not shown.

BIN
infrastructure/sandbox/Data/lambda/elasticsearch/client/__pycache__/cluster.cpython-310.pyc
View file

Binary file not shown.

BIN
infrastructure/sandbox/Data/lambda/elasticsearch/client/__pycache__/indices.cpython-310.pyc
View file

Binary file not shown.

BIN
infrastructure/sandbox/Data/lambda/elasticsearch/client/__pycache__/ingest.cpython-310.pyc
View file

Binary file not shown.

BIN
infrastructure/sandbox/Data/lambda/elasticsearch/client/__pycache__/nodes.cpython-310.pyc
View file

Binary file not shown.

BIN
infrastructure/sandbox/Data/lambda/elasticsearch/client/__pycache__/remote.cpython-310.pyc
View file

Binary file not shown.

BIN
infrastructure/sandbox/Data/lambda/elasticsearch/client/__pycache__/snapshot.cpython-310.pyc
View file

Binary file not shown.

BIN
infrastructure/sandbox/Data/lambda/elasticsearch/client/__pycache__/tasks.cpython-310.pyc
View file

Binary file not shown.

BIN
infrastructure/sandbox/Data/lambda/elasticsearch/client/__pycache__/utils.cpython-310.pyc
View file

Binary file not shown.

470
infrastructure/sandbox/Data/lambda/elasticsearch/client/cat.py
View file

@ -1,470 +0,0 @@
# Licensed to Elasticsearch B.V. under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch B.V. licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
from .utils import NamespacedClient, query_params, _make_path
class CatClient(NamespacedClient):
@query_params("format", "h", "help", "local", "master_timeout", "s", "v")
def aliases(self, name=None, params=None):
"""
`<http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-alias.html>`_
:arg name: A comma-separated list of alias names to return
:arg format: a short version of the Accept header, e.g. json, yaml
:arg h: Comma-separated list of column names to display
:arg help: Return help information, default False
:arg local: Return local information, do not retrieve the state from
master node (default: false)
:arg master_timeout: Explicit operation timeout for connection to master
node
:arg s: Comma-separated list of column names or column aliases to sort
by
:arg v: Verbose mode. Display column headers, default False
"""
return self.transport.perform_request(
"GET", _make_path("_cat", "aliases", name), params=params
)
@query_params("bytes", "format", "h", "help", "local", "master_timeout", "s", "v")
def allocation(self, node_id=None, params=None):
"""
Allocation provides a snapshot of how shards have located around the
cluster and the state of disk usage.
`<http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-allocation.html>`_
:arg node_id: A comma-separated list of node IDs or names to limit the
returned information
:arg bytes: The unit in which to display byte values, valid choices are:
'b', 'k', 'kb', 'm', 'mb', 'g', 'gb', 't', 'tb', 'p', 'pb'
:arg format: a short version of the Accept header, e.g. json, yaml
:arg h: Comma-separated list of column names to display
:arg help: Return help information, default False
:arg local: Return local information, do not retrieve the state from
master node (default: false)
:arg master_timeout: Explicit operation timeout for connection to master
node
:arg s: Comma-separated list of column names or column aliases to sort
by
:arg v: Verbose mode. Display column headers, default False
"""
return self.transport.perform_request(
"GET", _make_path("_cat", "allocation", node_id), params=params
)
@query_params("format", "h", "help", "local", "master_timeout", "s", "v")
def count(self, index=None, params=None):
"""
Count provides quick access to the document count of the entire cluster,
or individual indices.
`<http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-count.html>`_
:arg index: A comma-separated list of index names to limit the returned
information
:arg format: a short version of the Accept header, e.g. json, yaml
:arg h: Comma-separated list of column names to display
:arg help: Return help information, default False
:arg local: Return local information, do not retrieve the state from
master node (default: false)
:arg master_timeout: Explicit operation timeout for connection to master
node
:arg s: Comma-separated list of column names or column aliases to sort
by
:arg v: Verbose mode. Display column headers, default False
"""
return self.transport.perform_request(
"GET", _make_path("_cat", "count", index), params=params
)
@query_params("bytes", "format", "h", "help", "local", "master_timeout", "s", "v")
def fielddata(self, fields=None, params=None):
"""
`<http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-fielddata.html>`_
:arg fields: A comma-separated list of fields to return the fielddata
size
:arg bytes: The unit in which to display byte values, valid choices are:
'b', 'k', 'kb', 'm', 'mb', 'g', 'gb', 't', 'tb', 'p', 'pb'
:arg format: a short version of the Accept header, e.g. json, yaml
:arg h: Comma-separated list of column names to display
:arg help: Return help information, default False
:arg local: Return local information, do not retrieve the state from
master node (default: false)
:arg master_timeout: Explicit operation timeout for connection to master
node
:arg s: Comma-separated list of column names or column aliases to sort
by
:arg v: Verbose mode. Display column headers, default False
"""
return self.transport.perform_request(
"GET", _make_path("_cat", "fielddata", fields), params=params
)
@query_params("format", "h", "help", "local", "master_timeout", "s", "ts", "v")
def health(self, params=None):
"""
health is a terse, one-line representation of the same information from
:meth:`~elasticsearch.client.cluster.ClusterClient.health` API
`<http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-health.html>`_
:arg format: a short version of the Accept header, e.g. json, yaml
:arg h: Comma-separated list of column names to display
:arg help: Return help information, default False
:arg local: Return local information, do not retrieve the state from
master node (default: false)
:arg master_timeout: Explicit operation timeout for connection to master
node
:arg s: Comma-separated list of column names or column aliases to sort
by
:arg ts: Set to false to disable timestamping, default True
:arg v: Verbose mode. Display column headers, default False
"""
return self.transport.perform_request("GET", "/_cat/health", params=params)
@query_params("help", "s")
def help(self, params=None):
"""
A simple help for the cat api.
`<http://www.elastic.co/guide/en/elasticsearch/reference/master/cat.html>`_
:arg help: Return help information, default False
:arg s: Comma-separated list of column names or column aliases to sort
by
"""
return self.transport.perform_request("GET", "/_cat", params=params)
@query_params(
"bytes",
"format",
"h",
"health",
"help",
"local",
"master_timeout",
"pri",
"s",
"v",
)
def indices(self, index=None, params=None):
"""
The indices command provides a cross-section of each index.
`<http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-indices.html>`_
:arg index: A comma-separated list of index names to limit the returned
information
:arg bytes: The unit in which to display byte values, valid choices are:
'b', 'k', 'm', 'g'
:arg format: a short version of the Accept header, e.g. json, yaml
:arg h: Comma-separated list of column names to display
:arg health: A health status ("green", "yellow", or "red" to filter only
indices matching the specified health status, default None, valid
choices are: 'green', 'yellow', 'red'
:arg help: Return help information, default False
:arg local: Return local information, do not retrieve the state from
master node (default: false)
:arg master_timeout: Explicit operation timeout for connection to master
node
:arg pri: Set to true to return stats only for primary shards, default
False
:arg s: Comma-separated list of column names or column aliases to sort
by
:arg v: Verbose mode. Display column headers, default False
"""
return self.transport.perform_request(
"GET", _make_path("_cat", "indices", index), params=params
)
@query_params("format", "h", "help", "local", "master_timeout", "s", "v")
def master(self, params=None):
"""
Displays the master's node ID, bound IP address, and node name.
`<http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-master.html>`_
:arg format: a short version of the Accept header, e.g. json, yaml
:arg h: Comma-separated list of column names to display
:arg help: Return help information, default False
:arg local: Return local information, do not retrieve the state from
master node (default: false)
:arg master_timeout: Explicit operation timeout for connection to master
node
:arg s: Comma-separated list of column names or column aliases to sort
by
:arg v: Verbose mode. Display column headers, default False
"""
return self.transport.perform_request("GET", "/_cat/master", params=params)
@query_params("format", "h", "help", "local", "master_timeout", "s", "v")
def nodeattrs(self, params=None):
"""
`<http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-nodeattrs.html>`_
:arg format: a short version of the Accept header, e.g. json, yaml
:arg h: Comma-separated list of column names to display
:arg help: Return help information, default False
:arg local: Return local information, do not retrieve the state from
master node (default: false)
:arg master_timeout: Explicit operation timeout for connection to master
node
:arg s: Comma-separated list of column names or column aliases to sort
by
:arg v: Verbose mode. Display column headers, default False
"""
return self.transport.perform_request("GET", "/_cat/nodeattrs", params=params)
@query_params("format", "full_id", "h", "help", "local", "master_timeout", "s", "v")
def nodes(self, params=None):
"""
The nodes command shows the cluster topology.
`<http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-nodes.html>`_
:arg format: a short version of the Accept header, e.g. json, yaml
:arg full_id: Return the full node ID instead of the shortened version
(default: false)
:arg h: Comma-separated list of column names to display
:arg help: Return help information, default False
:arg local: Return local information, do not retrieve the state from
master node (default: false)
:arg master_timeout: Explicit operation timeout for connection to master
node
:arg s: Comma-separated list of column names or column aliases to sort
by
:arg v: Verbose mode. Display column headers, default False
"""
return self.transport.perform_request("GET", "/_cat/nodes", params=params)
@query_params("format", "h", "help", "local", "master_timeout", "s", "v")
def pending_tasks(self, params=None):
"""
`<http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-pending-tasks.html>`_
:arg format: a short version of the Accept header, e.g. json, yaml
:arg h: Comma-separated list of column names to display
:arg help: Return help information, default False
:arg local: Return local information, do not retrieve the state from
master node (default: false)
:arg master_timeout: Explicit operation timeout for connection to master
node
:arg s: Comma-separated list of column names or column aliases to sort
by
:arg v: Verbose mode. Display column headers, default False
"""
return self.transport.perform_request(
"GET", "/_cat/pending_tasks", params=params
)
@query_params("format", "h", "help", "local", "master_timeout", "s", "v")
def plugins(self, params=None):
"""
`<http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-plugins.html>`_
:arg format: a short version of the Accept header, e.g. json, yaml
:arg h: Comma-separated list of column names to display
:arg help: Return help information, default False
:arg local: Return local information, do not retrieve the state from
master node (default: false)
:arg master_timeout: Explicit operation timeout for connection to master
node
:arg s: Comma-separated list of column names or column aliases to sort
by
:arg v: Verbose mode. Display column headers, default False
"""
return self.transport.perform_request("GET", "/_cat/plugins", params=params)
@query_params("bytes", "format", "h", "help", "master_timeout", "s", "v")
def recovery(self, index=None, params=None):
"""
`<http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-recovery.html>`_
:arg index: A comma-separated list of index names to limit the returned
information
:arg bytes: The unit in which to display byte values, valid choices are:
'b', 'k', 'kb', 'm', 'mb', 'g', 'gb', 't', 'tb', 'p', 'pb'
:arg format: a short version of the Accept header, e.g. json, yaml
:arg h: Comma-separated list of column names to display
:arg help: Return help information, default False
:arg master_timeout: Explicit operation timeout for connection to master
node
:arg s: Comma-separated list of column names or column aliases to sort
by
:arg v: Verbose mode. Display column headers, default False
"""
return self.transport.perform_request(
"GET", _make_path("_cat", "recovery", index), params=params
)
@query_params("format", "h", "help", "local", "master_timeout", "s", "v")
def repositories(self, params=None):
"""
`<http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-repositories.html>`_
:arg format: a short version of the Accept header, e.g. json, yaml
:arg h: Comma-separated list of column names to display
:arg help: Return help information, default False
:arg local: Return local information, do not retrieve the state from
master node, default False
:arg master_timeout: Explicit operation timeout for connection to master
node
:arg s: Comma-separated list of column names or column aliases to sort
by
:arg v: Verbose mode. Display column headers, default False
"""
return self.transport.perform_request(
"GET", "/_cat/repositories", params=params
)
@query_params("bytes", "format", "h", "help", "s", "v")
def segments(self, index=None, params=None):
"""
`<http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-segments.html>`_
:arg index: A comma-separated list of index names to limit the returned
information
:arg bytes: The unit in which to display byte values, valid choices are:
'b', 'k', 'kb', 'm', 'mb', 'g', 'gb', 't', 'tb', 'p', 'pb'
:arg format: a short version of the Accept header, e.g. json, yaml
:arg h: Comma-separated list of column names to display
:arg help: Return help information, default False
:arg s: Comma-separated list of column names or column aliases to sort
by
:arg v: Verbose mode. Display column headers, default False
"""
return self.transport.perform_request(
"GET", _make_path("_cat", "segments", index), params=params
)
@query_params("bytes", "format", "h", "help", "local", "master_timeout", "s", "v")
def shards(self, index=None, params=None):
"""
`<http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-shards.html>`_
:arg index: A comma-separated list of index names to limit the returned
information
:arg bytes: The unit in which to display byte values, valid choices are:
'b', 'k', 'kb', 'm', 'mb', 'g', 'gb', 't', 'tb', 'p', 'pb'
:arg format: a short version of the Accept header, e.g. json, yaml
:arg h: Comma-separated list of column names to display
:arg help: Return help information, default False
:arg local: Return local information, do not retrieve the state from
master node (default: false)
:arg master_timeout: Explicit operation timeout for connection to master
node
:arg s: Comma-separated list of column names or column aliases to sort
by
:arg v: Verbose mode. Display column headers, default False
"""
return self.transport.perform_request(
"GET", _make_path("_cat", "shards", index), params=params
)
@query_params(
"format", "h", "help", "ignore_unavailable", "master_timeout", "s", "v"
)
def snapshots(self, repository=None, params=None):
"""
`<http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-snapshots.html>`_
:arg repository: Name of repository from which to fetch the snapshot
information
:arg format: a short version of the Accept header, e.g. json, yaml
:arg h: Comma-separated list of column names to display
:arg help: Return help information, default False
:arg ignore_unavailable: Set to true to ignore unavailable snapshots,
default False
:arg master_timeout: Explicit operation timeout for connection to master
node
:arg s: Comma-separated list of column names or column aliases to sort
by
:arg v: Verbose mode. Display column headers, default False
"""
return self.transport.perform_request(
"GET", _make_path("_cat", "snapshots", repository), params=params
)
@query_params(
"actions", "detailed", "format", "h", "help", "node_id", "parent_task", "s", "v"
)
def tasks(self, params=None):
"""
`<http://www.elastic.co/guide/en/elasticsearch/reference/master/tasks.html>`_
:arg actions: A comma-separated list of actions that should be returned.
Leave empty to return all.
:arg detailed: Return detailed task information (default: false)
:arg format: a short version of the Accept header, e.g. json, yaml
:arg h: Comma-separated list of column names to display
:arg help: Return help information, default False
:arg node_id: A comma-separated list of node IDs or names to limit the
returned information; use `_local` to return information from the
node you're connecting to, leave empty to get information from all
nodes
:arg parent_task: Return tasks with specified parent task id. Set to -1
to return all.
:arg s: Comma-separated list of column names or column aliases to sort
by
:arg v: Verbose mode. Display column headers, default False
"""
return self.transport.perform_request("GET", "/_cat/tasks", params=params)
@query_params("format", "h", "help", "local", "master_timeout", "s", "v")
def templates(self, name=None, params=None):
"""
`<http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-templates.html>`_
:arg name: A pattern that returned template names must match
:arg format: a short version of the Accept header, e.g. json, yaml
:arg h: Comma-separated list of column names to display
:arg help: Return help information, default False
:arg local: Return local information, do not retrieve the state from
master node (default: false)
:arg master_timeout: Explicit operation timeout for connection to master
node
:arg s: Comma-separated list of column names or column aliases to sort
by
:arg v: Verbose mode. Display column headers, default False
"""
return self.transport.perform_request(
"GET", _make_path("_cat", "templates", name), params=params
)
@query_params("format", "h", "help", "local", "master_timeout", "s", "size", "v")
def thread_pool(self, thread_pool_patterns=None, params=None):
"""
`<http://www.elastic.co/guide/en/elasticsearch/reference/master/cat-thread-pool.html>`_
:arg thread_pool_patterns: A comma-separated list of regular-expressions
to filter the thread pools in the output
:arg format: a short version of the Accept header, e.g. json, yaml
:arg h: Comma-separated list of column names to display
:arg help: Return help information, default False
:arg local: Return local information, do not retrieve the state from
master node (default: false)
:arg master_timeout: Explicit operation timeout for connection to master
node
:arg s: Comma-separated list of column names or column aliases to sort
by
:arg size: The multiplier in which to display values, valid choices are:
'', 'k', 'm', 'g', 't', 'p'
:arg v: Verbose mode. Display column headers, default False
"""
return self.transport.perform_request(
"GET",
_make_path("_cat", "thread_pool", thread_pool_patterns),
params=params,
)

221
infrastructure/sandbox/Data/lambda/elasticsearch/client/cluster.py
View file

@ -1,221 +0,0 @@
# Licensed to Elasticsearch B.V. under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch B.V. licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
from .utils import NamespacedClient, query_params, _make_path
class ClusterClient(NamespacedClient):
@query_params(
"level",
"local",
"master_timeout",
"timeout",
"wait_for_active_shards",
"wait_for_events",
"wait_for_no_relocating_shards",
"wait_for_nodes",
"wait_for_status",
"wait_for_no_initializing_shards",
)
def health(self, index=None, params=None):
"""
Get a very simple status on the health of the cluster.
`<http://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-health.html>`_
:arg index: Limit the information returned to a specific index
:arg level: Specify the level of detail for returned information,
default 'cluster', valid choices are: 'cluster', 'indices', 'shards'
:arg local: Return local information, do not retrieve the state from
master node (default: false)
:arg master_timeout: Explicit operation timeout for connection to master
node
:arg timeout: Explicit operation timeout
:arg wait_for_active_shards: Wait until the specified number of shards
is active
:arg wait_for_events: Wait until all currently queued events with the
given priority are processed, valid choices are: 'immediate',
'urgent', 'high', 'normal', 'low', 'languid'
:arg wait_for_no_relocating_shards: Whether to wait until there are no
relocating shards in the cluster
:arg wait_for_nodes: Wait until the specified number of nodes is
available
:arg wait_for_status: Wait until cluster is in a specific state, default
None, valid choices are: 'green', 'yellow', 'red'
"""
return self.transport.perform_request(
"GET", _make_path("_cluster", "health", index), params=params
)
@query_params("local", "master_timeout")
def pending_tasks(self, params=None):
"""
The pending cluster tasks API returns a list of any cluster-level
changes (e.g. create index, update mapping, allocate or fail shard)
which have not yet been executed.
`<http://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-pending.html>`_
:arg local: Return local information, do not retrieve the state from
master node (default: false)
:arg master_timeout: Specify timeout for connection to master
"""
return self.transport.perform_request(
"GET", "/_cluster/pending_tasks", params=params
)
@query_params(
"allow_no_indices",
"expand_wildcards",
"flat_settings",
"ignore_unavailable",
"local",
"master_timeout",
"wait_for_metadata_version",
"wait_for_timeout",
)
def state(self, metric=None, index=None, params=None):
"""
Get a comprehensive state information of the whole cluster.
`<http://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-state.html>`_
:arg metric: Limit the information returned to the specified metrics
:arg index: A comma-separated list of index names; use `_all` or empty
string to perform the operation on all indices
:arg allow_no_indices: Whether to ignore if a wildcard indices
expression resolves into no concrete indices. (This includes `_all`
string or when no indices have been specified)
:arg expand_wildcards: Whether to expand wildcard expression to concrete
indices that are open, closed or both., default 'open', valid
choices are: 'open', 'closed', 'none', 'all'
:arg flat_settings: Return settings in flat format (default: false)
:arg ignore_unavailable: Whether specified concrete indices should be
ignored when unavailable (missing or closed)
:arg local: Return local information, do not retrieve the state from
master node (default: false)
:arg master_timeout: Specify timeout for connection to master
:arg wait_for_metadata_version: Wait for the metadata version to be
equal or greater than the specified metadata version
:arg wait_for_timeout: The maximum time to wait for
wait_for_metadata_version before timing out
"""
if index and not metric:
metric = "_all"
return self.transport.perform_request(
"GET", _make_path("_cluster", "state", metric, index), params=params
)
@query_params("flat_settings", "timeout")
def stats(self, node_id=None, params=None):
"""
The Cluster Stats API allows to retrieve statistics from a cluster wide
perspective. The API returns basic index metrics and information about
the current nodes that form the cluster.
`<http://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-stats.html>`_
:arg node_id: A comma-separated list of node IDs or names to limit the
returned information; use `_local` to return information from the
node you're connecting to, leave empty to get information from all
nodes
:arg flat_settings: Return settings in flat format (default: false)
:arg timeout: Explicit operation timeout
"""
url = "/_cluster/stats"
if node_id:
url = _make_path("_cluster/stats/nodes", node_id)
return self.transport.perform_request("GET", url, params=params)
@query_params(
"dry_run", "explain", "master_timeout", "metric", "retry_failed", "timeout"
)
def reroute(self, body=None, params=None):
"""
Explicitly execute a cluster reroute allocation command including specific commands.
`<http://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-reroute.html>`_
:arg body: The definition of `commands` to perform (`move`, `cancel`,
`allocate`)
:arg dry_run: Simulate the operation only and return the resulting state
:arg explain: Return an explanation of why the commands can or cannot be
executed
:arg master_timeout: Explicit operation timeout for connection to master
node
:arg metric: Limit the information returned to the specified metrics.
Defaults to all but metadata, valid choices are: '_all', 'blocks',
'metadata', 'nodes', 'routing_table', 'master_node', 'version'
:arg retry_failed: Retries allocation of shards that are blocked due to
too many subsequent allocation failures
:arg timeout: Explicit operation timeout
"""
return self.transport.perform_request(
"POST", "/_cluster/reroute", params=params, body=body
)
@query_params("flat_settings", "include_defaults", "master_timeout", "timeout")
def get_settings(self, params=None):
"""
Get cluster settings.
`<http://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-update-settings.html>`_
:arg flat_settings: Return settings in flat format (default: false)
:arg include_defaults: Whether to return all default clusters setting.,
default False
:arg master_timeout: Explicit operation timeout for connection to master
node
:arg timeout: Explicit operation timeout
"""
return self.transport.perform_request(
"GET", "/_cluster/settings", params=params
)
@query_params("flat_settings", "master_timeout", "timeout")
def put_settings(self, body=None, params=None):
"""
Update cluster wide specific settings.
`<http://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-update-settings.html>`_
:arg body: The settings to be updated. Can be either `transient` or
`persistent` (survives cluster restart).
:arg flat_settings: Return settings in flat format (default: false)
:arg master_timeout: Explicit operation timeout for connection to master
node
:arg timeout: Explicit operation timeout
"""
return self.transport.perform_request(
"PUT", "/_cluster/settings", params=params, body=body
)
@query_params("include_disk_info", "include_yes_decisions")
def allocation_explain(self, body=None, params=None):
"""
`<http://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-allocation-explain.html>`_
:arg body: The index, shard, and primary flag to explain. Empty means
'explain the first unassigned shard'
:arg include_disk_info: Return information about disk usage and shard
sizes (default: false)
:arg include_yes_decisions: Return 'YES' decisions in explanation
(default: false)
"""
return self.transport.perform_request(
"GET", "/_cluster/allocation/explain", params=params, body=body
)
@query_params()
def remote_info(self, params=None):
"""
`<http://www.elastic.co/guide/en/elasticsearch/reference/master/cluster-remote-info.html>`_
"""
return self.transport.perform_request("GET", "/_remote/info", params=params)

1105
infrastructure/sandbox/Data/lambda/elasticsearch/client/indices.py
View file

File diff suppressed because it is too large Load diff

95
infrastructure/sandbox/Data/lambda/elasticsearch/client/ingest.py
View file

@ -1,95 +0,0 @@
# Licensed to Elasticsearch B.V. under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch B.V. licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
from .utils import NamespacedClient, query_params, _make_path, SKIP_IN_PATH
class IngestClient(NamespacedClient):
@query_params("master_timeout")
def get_pipeline(self, id=None, params=None):
"""
`<https://www.elastic.co/guide/en/elasticsearch/plugins/current/ingest.html>`_
:arg id: Comma separated list of pipeline ids. Wildcards supported
:arg master_timeout: Explicit operation timeout for connection to master
node
"""
return self.transport.perform_request(
"GET", _make_path("_ingest", "pipeline", id), params=params
)
@query_params("master_timeout", "timeout")
def put_pipeline(self, id, body, params=None):
"""
`<https://www.elastic.co/guide/en/elasticsearch/plugins/current/ingest.html>`_
:arg id: Pipeline ID
:arg body: The ingest definition
:arg master_timeout: Explicit operation timeout for connection to master
node
:arg timeout: Explicit operation timeout
"""
for param in (id, body):
if param in SKIP_IN_PATH:
raise ValueError("Empty value passed for a required argument.")
return self.transport.perform_request(
"PUT", _make_path("_ingest", "pipeline", id), params=params, body=body
)
@query_params("master_timeout", "timeout")
def delete_pipeline(self, id, params=None):
"""
`<https://www.elastic.co/guide/en/elasticsearch/plugins/current/ingest.html>`_
:arg id: Pipeline ID
:arg master_timeout: Explicit operation timeout for connection to master
node
:arg timeout: Explicit operation timeout
"""
if id in SKIP_IN_PATH:
raise ValueError("Empty value passed for a required argument 'id'.")
return self.transport.perform_request(
"DELETE", _make_path("_ingest", "pipeline", id), params=params
)
@query_params("verbose")
def simulate(self, body, id=None, params=None):
"""
`<https://www.elastic.co/guide/en/elasticsearch/plugins/current/ingest.html>`_
:arg body: The simulate definition
:arg id: Pipeline ID
:arg verbose: Verbose mode. Display data output for each processor in
executed pipeline, default False
"""
if body in SKIP_IN_PATH:
raise ValueError("Empty value passed for a required argument 'body'.")
return self.transport.perform_request(
"GET",
_make_path("_ingest", "pipeline", id, "_simulate"),
params=params,
body=body,
)
@query_params()
def processor_grok(self, params=None):
"""
`<https://www.elastic.co/guide/en/elasticsearch/reference/master/grok-processor.html#grok-processor-rest-get>`_
"""
return self.transport.perform_request(
"GET", "/_ingest/processor/grok", params=params
)

154
infrastructure/sandbox/Data/lambda/elasticsearch/client/nodes.py
View file

@ -1,154 +0,0 @@
# Licensed to Elasticsearch B.V. under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch B.V. licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
from .utils import NamespacedClient, query_params, _make_path
class NodesClient(NamespacedClient):
@query_params("flat_settings", "timeout")
def info(self, node_id=None, metric=None, params=None):
"""
The cluster nodes info API allows to retrieve one or more (or all) of
the cluster nodes information.
`<https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-nodes-info.html>`_
:arg node_id: A comma-separated list of node IDs or names to limit the
returned information; use `_local` to return information from the
node you're connecting to, leave empty to get information from all
nodes
:arg metric: A comma-separated list of metrics you wish returned. Leave
empty to return all.
:arg flat_settings: Return settings in flat format (default: false)
:arg timeout: Explicit operation timeout
"""
return self.transport.perform_request(
"GET", _make_path("_nodes", node_id, metric), params=params
)
@query_params(
"completion_fields",
"fielddata_fields",
"fields",
"groups",
"include_segment_file_sizes",
"level",
"timeout",
"types",
)
def stats(self, node_id=None, metric=None, index_metric=None, params=None):
"""
The cluster nodes stats API allows to retrieve one or more (or all) of
the cluster nodes statistics.
`<https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-nodes-stats.html>`_
:arg node_id: A comma-separated list of node IDs or names to limit the
returned information; use `_local` to return information from the
node you're connecting to, leave empty to get information from all
nodes
:arg metric: Limit the information returned to the specified metrics
:arg index_metric: Limit the information returned for `indices` metric
to the specific index metrics. Isn't used if `indices` (or `all`)
metric isn't specified.
:arg completion_fields: A comma-separated list of fields for `fielddata`
and `suggest` index metric (supports wildcards)
:arg fielddata_fields: A comma-separated list of fields for `fielddata`
index metric (supports wildcards)
:arg fields: A comma-separated list of fields for `fielddata` and
`completion` index metric (supports wildcards)
:arg groups: A comma-separated list of search groups for `search` index
metric
:arg include_segment_file_sizes: Whether to report the aggregated disk
usage of each one of the Lucene index files (only applies if segment
stats are requested), default False
:arg level: Return indices stats aggregated at index, node or shard
level, default 'node', valid choices are: 'indices', 'node',
'shards'
:arg timeout: Explicit operation timeout
:arg types: A comma-separated list of document types for the `indexing`
index metric
"""
return self.transport.perform_request(
"GET",
_make_path("_nodes", node_id, "stats", metric, index_metric),
params=params,
)
@query_params(
"type", "ignore_idle_threads", "interval", "snapshots", "threads", "timeout"
)
def hot_threads(self, node_id=None, params=None):
"""
An API allowing to get the current hot threads on each node in the cluster.
`<https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-nodes-hot-threads.html>`_
:arg node_id: A comma-separated list of node IDs or names to limit the
returned information; use `_local` to return information from the
node you're connecting to, leave empty to get information from all
nodes
:arg type: The type to sample (default: cpu), valid choices are:
'cpu', 'wait', 'block'
:arg ignore_idle_threads: Don't show threads that are in known-idle
places, such as waiting on a socket select or pulling from an empty
task queue (default: true)
:arg interval: The interval for the second sampling of threads
:arg snapshots: Number of samples of thread stacktrace (default: 10)
:arg threads: Specify the number of threads to provide information for
(default: 3)
:arg timeout: Explicit operation timeout
"""
# avoid python reserved words
if params and "type_" in params:
params["type"] = params.pop("type_")
return self.transport.perform_request(
"GET", _make_path("_cluster", "nodes", node_id, "hotthreads"), params=params
)
@query_params("human", "timeout")
def usage(self, node_id=None, metric=None, params=None):
"""
The cluster nodes usage API allows to retrieve information on the usage
of features for each node.
`<http://www.elastic.co/guide/en/elasticsearch/reference/master/cluster-nodes-usage.html>`_
:arg node_id: A comma-separated list of node IDs or names to limit the
returned information; use `_local` to return information from the
node you're connecting to, leave empty to get information from all
nodes
:arg metric: Limit the information returned to the specified metrics
:arg human: Whether to return time and byte values in human-readable
format., default False
:arg timeout: Explicit operation timeout
"""
return self.transport.perform_request(
"GET", _make_path("_nodes", node_id, "usage", metric), params=params
)
@query_params("timeout")
def reload_secure_settings(self, node_id=None, params=None):
"""
`<https://www.elastic.co/guide/en/elasticsearch/reference/6.x/secure-settings.html#reloadable-secure-settings>`_
:arg node_id: A comma-separated list of node IDs to span the
reload/reinit call. Should stay empty because reloading usually
involves all cluster nodes.
:arg timeout: Explicit operation timeout
"""
return self.transport.perform_request(
"POST",
_make_path("_nodes", node_id, "reload_secure_settings"),
params=params,
)

27
infrastructure/sandbox/Data/lambda/elasticsearch/client/remote.py
View file

@ -1,27 +0,0 @@
# Licensed to Elasticsearch B.V. under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch B.V. licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
from .utils import NamespacedClient, query_params
class RemoteClient(NamespacedClient):
@query_params()
def info(self, params=None):
"""
`<http://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-remote-info.html>`_
"""
return self.transport.perform_request("GET", "/_remote/info", params=params)

200
infrastructure/sandbox/Data/lambda/elasticsearch/client/snapshot.py
View file

@ -1,200 +0,0 @@
# Licensed to Elasticsearch B.V. under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch B.V. licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
from .utils import NamespacedClient, query_params, _make_path, SKIP_IN_PATH
class SnapshotClient(NamespacedClient):
@query_params("master_timeout", "wait_for_completion")
def create(self, repository, snapshot, body=None, params=None):
"""
Create a snapshot in repository
`<http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-snapshots.html>`_
:arg repository: A repository name
:arg snapshot: A snapshot name
:arg body: The snapshot definition
:arg master_timeout: Explicit operation timeout for connection to master
node
:arg wait_for_completion: Should this request wait until the operation
has completed before returning, default False
"""
for param in (repository, snapshot):
if param in SKIP_IN_PATH:
raise ValueError("Empty value passed for a required argument.")
return self.transport.perform_request(
"PUT",
_make_path("_snapshot", repository, snapshot),
params=params,
body=body,
)
@query_params("master_timeout")
def delete(self, repository, snapshot, params=None):
"""
Deletes a snapshot from a repository.
`<http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-snapshots.html>`_
:arg repository: A repository name
:arg snapshot: A snapshot name
:arg master_timeout: Explicit operation timeout for connection to master
node
"""
for param in (repository, snapshot):
if param in SKIP_IN_PATH:
raise ValueError("Empty value passed for a required argument.")
return self.transport.perform_request(
"DELETE", _make_path("_snapshot", repository, snapshot), params=params
)
@query_params("ignore_unavailable", "master_timeout", "verbose")
def get(self, repository, snapshot, params=None):
"""
Retrieve information about a snapshot.
`<http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-snapshots.html>`_
:arg repository: A repository name
:arg snapshot: A comma-separated list of snapshot names
:arg ignore_unavailable: Whether to ignore unavailable snapshots,
defaults to false which means a NotFoundError `snapshot_missing_exception` is thrown
:arg master_timeout: Explicit operation timeout for connection to master
node
:arg verbose: Whether to show verbose snapshot info or only show the
basic info found in the repository index blob
"""
for param in (repository, snapshot):
if param in SKIP_IN_PATH:
raise ValueError("Empty value passed for a required argument.")
return self.transport.perform_request(
"GET", _make_path("_snapshot", repository, snapshot), params=params
)
@query_params("master_timeout", "timeout")
def delete_repository(self, repository, params=None):
"""
Removes a shared file system repository.
`<http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-snapshots.html>`_
:arg repository: A comma-separated list of repository names
:arg master_timeout: Explicit operation timeout for connection to master
node
:arg timeout: Explicit operation timeout
"""
if repository in SKIP_IN_PATH:
raise ValueError("Empty value passed for a required argument 'repository'.")
return self.transport.perform_request(
"DELETE", _make_path("_snapshot", repository), params=params
)
@query_params("local", "master_timeout")
def get_repository(self, repository=None, params=None):
"""
Return information about registered repositories.
`<http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-snapshots.html>`_
:arg repository: A comma-separated list of repository names
:arg local: Return local information, do not retrieve the state from
master node (default: false)
:arg master_timeout: Explicit operation timeout for connection to master
node
"""
return self.transport.perform_request(
"GET", _make_path("_snapshot", repository), params=params
)
@query_params("master_timeout", "timeout", "verify")
def create_repository(self, repository, body, params=None):
"""
Registers a shared file system repository.
`<http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-snapshots.html>`_
:arg repository: A repository name
:arg body: The repository definition
:arg master_timeout: Explicit operation timeout for connection to master
node
:arg timeout: Explicit operation timeout
:arg verify: Whether to verify the repository after creation
"""
for param in (repository, body):
if param in SKIP_IN_PATH:
raise ValueError("Empty value passed for a required argument.")
return self.transport.perform_request(
"PUT", _make_path("_snapshot", repository), params=params, body=body
)
@query_params("master_timeout", "wait_for_completion")
def restore(self, repository, snapshot, body=None, params=None):
"""
Restore a snapshot.
`<http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-snapshots.html>`_
:arg repository: A repository name
:arg snapshot: A snapshot name
:arg body: Details of what to restore
:arg master_timeout: Explicit operation timeout for connection to master
node
:arg wait_for_completion: Should this request wait until the operation
has completed before returning, default False
"""
for param in (repository, snapshot):
if param in SKIP_IN_PATH:
raise ValueError("Empty value passed for a required argument.")
return self.transport.perform_request(
"POST",
_make_path("_snapshot", repository, snapshot, "_restore"),
params=params,
body=body,
)
@query_params("ignore_unavailable", "master_timeout")
def status(self, repository=None, snapshot=None, params=None):
"""
Return information about all currently running snapshots. By specifying
a repository name, it's possible to limit the results to a particular
repository.
`<http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-snapshots.html>`_
:arg repository: A repository name
:arg snapshot: A comma-separated list of snapshot names
:arg ignore_unavailable: Whether to ignore unavailable snapshots,
defaults to false which means a NotFoundError `snapshot_missing_exception` is thrown
:arg master_timeout: Explicit operation timeout for connection to master
node
"""
return self.transport.perform_request(
"GET",
_make_path("_snapshot", repository, snapshot, "_status"),
params=params,
)
@query_params("master_timeout", "timeout")
def verify_repository(self, repository, params=None):
"""
Returns a list of nodes where repository was successfully verified or
an error message if verification process failed.
`<http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-snapshots.html>`_
:arg repository: A repository name
:arg master_timeout: Explicit operation timeout for connection to master
node
:arg timeout: Explicit operation timeout
"""
if repository in SKIP_IN_PATH:
raise ValueError("Empty value passed for a required argument 'repository'.")
return self.transport.perform_request(
"POST", _make_path("_snapshot", repository, "_verify"), params=params
)

86
infrastructure/sandbox/Data/lambda/elasticsearch/client/tasks.py
View file

@ -1,86 +0,0 @@
# Licensed to Elasticsearch B.V. under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch B.V. licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
from .utils import NamespacedClient, query_params, _make_path
class TasksClient(NamespacedClient):
@query_params(
"actions",
"detailed",
"group_by",
"nodes",
"parent_task_id",
"wait_for_completion",
"timeout",
)
def list(self, params=None):
"""
`<http://www.elastic.co/guide/en/elasticsearch/reference/current/tasks.html>`_
:arg actions: A comma-separated list of actions that should be returned.
Leave empty to return all.
:arg detailed: Return detailed task information (default: false)
:arg group_by: Group tasks by nodes or parent/child relationships,
default 'nodes', valid choices are: 'nodes', 'parents'
:arg nodes: A comma-separated list of node IDs or names to limit the
returned information; use `_local` to return information from the
node you're connecting to, leave empty to get information from all
nodes
:arg parent_task_id: Return tasks with specified parent task id
(node_id:task_number). Set to -1 to return all.
:arg wait_for_completion: Wait for the matching tasks to complete
(default: false)
:arg timeout: Maximum waiting time for `wait_for_completion`
"""
return self.transport.perform_request("GET", "/_tasks", params=params)
@query_params("actions", "nodes", "parent_task_id")
def cancel(self, task_id=None, params=None):
"""
`<http://www.elastic.co/guide/en/elasticsearch/reference/current/tasks.html>`_
:arg task_id: Cancel the task with specified task id
(node_id:task_number)
:arg actions: A comma-separated list of actions that should be
cancelled. Leave empty to cancel all.
:arg nodes: A comma-separated list of node IDs or names to limit the
returned information; use `_local` to return information from the
node you're connecting to, leave empty to get information from all
nodes
:arg parent_task_id: Cancel tasks with specified parent task id
(node_id:task_number). Set to -1 to cancel all.
"""
return self.transport.perform_request(
"POST", _make_path("_tasks", task_id, "_cancel"), params=params
)
@query_params("wait_for_completion", "timeout")
def get(self, task_id=None, params=None):
"""
Retrieve information for a particular task.
`<http://www.elastic.co/guide/en/elasticsearch/reference/current/tasks.html>`_
:arg task_id: Return the task with specified id (node_id:task_number)
:arg wait_for_completion: Wait for the matching tasks to complete
(default: false)
:arg timeout: Maximum waiting time for `wait_for_completion`
"""
return self.transport.perform_request(
"GET", _make_path("_tasks", task_id), params=params
)

122
infrastructure/sandbox/Data/lambda/elasticsearch/client/utils.py
View file

@ -1,122 +0,0 @@
# Licensed to Elasticsearch B.V. under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch B.V. licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
from __future__ import unicode_literals
import weakref
from datetime import date, datetime
from functools import wraps
from ..compat import string_types, quote, PY2
# parts of URL to be omitted
SKIP_IN_PATH = (None, "", b"", [], ())
def _escape(value):
"""
Escape a single value of a URL string or a query parameter. If it is a list
or tuple, turn it into a comma-separated string first.
"""
# make sequences into comma-separated stings
if isinstance(value, (list, tuple)):
value = ",".join(value)
# dates and datetimes into isoformat
elif isinstance(value, (date, datetime)):
value = value.isoformat()
# make bools into true/false strings
elif isinstance(value, bool):
value = str(value).lower()
# don't decode bytestrings
elif isinstance(value, bytes):
return value
# encode strings to utf-8
if isinstance(value, string_types):
if PY2 and isinstance(value, unicode): # noqa: F821
return value.encode("utf-8")
if not PY2 and isinstance(value, str):
return value.encode("utf-8")
return str(value)
def _make_path(*parts):
"""
Create a URL string from parts, omit all `None` values and empty strings.
Convert lists nad tuples to comma separated values.
"""
# TODO: maybe only allow some parts to be lists/tuples ?
return "/" + "/".join(
# preserve ',' and '*' in url for nicer URLs in logs
quote(_escape(p), b",*")
for p in parts
if p not in SKIP_IN_PATH
)
# parameters that apply to all methods
GLOBAL_PARAMS = ("pretty", "human", "error_trace", "format", "filter_path")
def query_params(*es_query_params):
"""
Decorator that pops all accepted parameters from method's kwargs and puts
them in the params argument.
"""
def _wrapper(func):
@wraps(func)
def _wrapped(*args, **kwargs):
params = {}
if "params" in kwargs:
params = kwargs.pop("params").copy()
for p in es_query_params + GLOBAL_PARAMS:
if p in kwargs:
v = kwargs.pop(p)
if v is not None:
params[p] = _escape(v)
# don't treat ignore and request_timeout as other params to avoid escaping
for p in ("ignore", "request_timeout"):
if p in kwargs:
params[p] = kwargs.pop(p)
return func(*args, params=params, **kwargs)
return _wrapped
return _wrapper
class NamespacedClient(object):
def __init__(self, client):
self.client = client
@property
def transport(self):
return self.client.transport
class AddonClient(NamespacedClient):
@classmethod
def infect_client(cls, client):
addon = cls(weakref.proxy(client))
setattr(client, cls.namespace, addon)
return client

64
infrastructure/sandbox/Data/lambda/elasticsearch/client/xpack/__init__.py
View file

@ -1,64 +0,0 @@
# Licensed to Elasticsearch B.V. under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch B.V. licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
from ..utils import NamespacedClient, query_params
from .graph import GraphClient
from .license import LicenseClient
from .monitoring import MonitoringClient
from .security import SecurityClient
from .watcher import WatcherClient
from .ml import MlClient
from .migration import MigrationClient
from .deprecation import DeprecationClient
class XPackClient(NamespacedClient):
namespace = "xpack"
def __init__(self, *args, **kwargs):
super(XPackClient, self).__init__(*args, **kwargs)
self.graph = GraphClient(self.client)
self.license = LicenseClient(self.client)
self.monitoring = MonitoringClient(self.client)
self.security = SecurityClient(self.client)
self.watcher = WatcherClient(self.client)
self.ml = MlClient(self.client)
self.migration = MigrationClient(self.client)
self.deprecation = DeprecationClient(self.client)
@query_params("categories", "human")
def info(self, params=None):
"""
Retrieve information about xpack, including build number/timestamp and license status
`<https://www.elastic.co/guide/en/elasticsearch/reference/current/info-api.html>`_
:arg categories: Comma-separated list of info categories. Can be any of:
build, license, features
:arg human: Presents additional info for humans (feature descriptions
and X-Pack tagline)
"""
return self.transport.perform_request("GET", "/_xpack", params=params)
@query_params("master_timeout")
def usage(self, params=None):
"""
Retrieve information about xpack features usage
:arg master_timeout: Specify timeout for watch write operation
"""
return self.transport.perform_request("GET", "/_xpack/usage", params=params)

BIN
infrastructure/sandbox/Data/lambda/elasticsearch/client/xpack/__pycache__/__init__.cpython-310.pyc
View file

Binary file not shown.

BIN
infrastructure/sandbox/Data/lambda/elasticsearch/client/xpack/__pycache__/deprecation.cpython-310.pyc
View file

Binary file not shown.

BIN
infrastructure/sandbox/Data/lambda/elasticsearch/client/xpack/__pycache__/graph.cpython-310.pyc
View file

Binary file not shown.

BIN
infrastructure/sandbox/Data/lambda/elasticsearch/client/xpack/__pycache__/license.cpython-310.pyc
View file

Binary file not shown.

BIN
infrastructure/sandbox/Data/lambda/elasticsearch/client/xpack/__pycache__/migration.cpython-310.pyc
View file

Binary file not shown.

BIN
infrastructure/sandbox/Data/lambda/elasticsearch/client/xpack/__pycache__/ml.cpython-310.pyc
View file

Binary file not shown.

BIN
infrastructure/sandbox/Data/lambda/elasticsearch/client/xpack/__pycache__/monitoring.cpython-310.pyc
View file

Binary file not shown.

BIN
infrastructure/sandbox/Data/lambda/elasticsearch/client/xpack/__pycache__/security.cpython-310.pyc
View file

Binary file not shown.

BIN
infrastructure/sandbox/Data/lambda/elasticsearch/client/xpack/__pycache__/watcher.cpython-310.pyc
View file

Binary file not shown.

33
infrastructure/sandbox/Data/lambda/elasticsearch/client/xpack/deprecation.py
View file

@ -1,33 +0,0 @@
# Licensed to Elasticsearch B.V. under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch B.V. licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
from ..utils import NamespacedClient, query_params, _make_path
class DeprecationClient(NamespacedClient):
@query_params()
def info(self, index=None, params=None):
"""
`<http://www.elastic.co/guide/en/migration/current/migration-api-deprecation.html>`_
:arg index: Index pattern
"""
return self.transport.perform_request(
"GET",
_make_path(index, "_xpack", "migration", "deprecations"),
params=params,
)

40
infrastructure/sandbox/Data/lambda/elasticsearch/client/xpack/graph.py
View file

@ -1,40 +0,0 @@
# Licensed to Elasticsearch B.V. under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch B.V. licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
from ..utils import NamespacedClient, query_params, _make_path
class GraphClient(NamespacedClient):
@query_params("routing", "timeout")
def explore(self, index=None, doc_type=None, body=None, params=None):
"""
`<https://www.elastic.co/guide/en/elasticsearch/reference/current/graph-explore-api.html>`_
:arg index: A comma-separated list of index names to search; use `_all`
or empty string to perform the operation on all indices
:arg doc_type: A comma-separated list of document types to search; leave
empty to perform the operation on all types
:arg body: Graph Query DSL
:arg routing: Specific routing value
:arg timeout: Explicit operation timeout
"""
return self.transport.perform_request(
"GET",
_make_path(index, doc_type, "_xpack", "graph", "_explore"),
params=params,
body=body,
)

58
infrastructure/sandbox/Data/lambda/elasticsearch/client/xpack/license.py
View file

@ -1,58 +0,0 @@
# Licensed to Elasticsearch B.V. under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch B.V. licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
from ..utils import (
NamespacedClient,
query_params,
)
class LicenseClient(NamespacedClient):
@query_params()
def delete(self, params=None):
"""
`<https://www.elastic.co/guide/en/x-pack/current/license-management.html>`_
"""
return self.transport.perform_request(
"DELETE", "/_xpack/license", params=params
)
@query_params("local")
def get(self, params=None):
"""
`<https://www.elastic.co/guide/en/x-pack/current/license-management.html>`_
:arg local: Return local information, do not retrieve the state from
master node (default: false)
"""
return self.transport.perform_request("GET", "/_xpack/license", params=params)
@query_params("acknowledge")
def post(self, body=None, params=None):
"""
`<https://www.elastic.co/guide/en/x-pack/current/license-management.html>`_
:arg body: licenses to be installed
:arg acknowledge: whether the user has acknowledged acknowledge messages
(default: false)
"""
return self.transport.perform_request(
"PUT", "/_xpack/license", params=params, body=body
)

61
infrastructure/sandbox/Data/lambda/elasticsearch/client/xpack/migration.py
View file

@ -1,61 +0,0 @@
# Licensed to Elasticsearch B.V. under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch B.V. licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
from ..utils import (
NamespacedClient,
query_params,
_make_path,
SKIP_IN_PATH,
)
class MigrationClient(NamespacedClient):
@query_params("allow_no_indices", "expand_wildcards", "ignore_unavailable")
def get_assistance(self, index=None, params=None):
"""
`<https://www.elastic.co/guide/en/elasticsearch/reference/current/migration-api-assistance.html>`_
:arg index: A comma-separated list of index names; use `_all` or empty
string to perform the operation on all indices
:arg allow_no_indices: Whether to ignore if a wildcard indices
expression resolves into no concrete indices. (This includes `_all`
string or when no indices have been specified)
:arg expand_wildcards: Whether to expand wildcard expression to concrete
indices that are open, closed or both., default 'open', valid
choices are: 'open', 'closed', 'none', 'all'
:arg ignore_unavailable: Whether specified concrete indices should be
ignored when unavailable (missing or closed)
"""
return self.transport.perform_request(
"GET", _make_path("_xpack", "migration", "assistance", index), params=params
)
@query_params("wait_for_completion")
def upgrade(self, index, params=None):
"""
`<https://www.elastic.co/guide/en/elasticsearch/reference/current/migration-api-upgrade.html>`_
:arg index: The name of the index
:arg wait_for_completion: Should the request block until the upgrade
operation is completed, default True
"""
if index in SKIP_IN_PATH:
raise ValueError("Empty value passed for a required argument 'index'.")
return self.transport.perform_request(
"POST", _make_path("_xpack", "migration", "upgrade", index), params=params
)

695
infrastructure/sandbox/Data/lambda/elasticsearch/client/xpack/ml.py
View file

@ -1,695 +0,0 @@
# Licensed to Elasticsearch B.V. under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch B.V. licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
from ..utils import NamespacedClient, query_params, _make_path, SKIP_IN_PATH
class MlClient(NamespacedClient):
@query_params("from_", "size")
def get_filters(self, filter_id=None, params=None):
"""
:arg filter_id: The ID of the filter to fetch
:arg from_: skips a number of filters
:arg size: specifies a max number of filters to get
"""
return self.transport.perform_request(
"GET", _make_path("_xpack", "ml", "filters", filter_id), params=params
)
@query_params()
def get_datafeeds(self, datafeed_id=None, params=None):
"""
`<http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-get-datafeed.html>`_
:arg datafeed_id: The ID of the datafeeds to fetch
"""
return self.transport.perform_request(
"GET", _make_path("_xpack", "ml", "datafeeds", datafeed_id), params=params
)
@query_params()
def get_datafeed_stats(self, datafeed_id=None, params=None):
"""
`<http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-get-datafeed-stats.html>`_
:arg datafeed_id: The ID of the datafeeds stats to fetch
"""
return self.transport.perform_request(
"GET",
_make_path("_xpack", "ml", "datafeeds", datafeed_id, "_stats"),
params=params,
)
@query_params(
"anomaly_score",
"desc",
"end",
"exclude_interim",
"expand",
"from_",
"size",
"sort",
"start",
)
def get_buckets(self, job_id, timestamp=None, body=None, params=None):
"""
`<http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-get-bucket.html>`_
:arg job_id: ID of the job to get bucket results from
:arg timestamp: The timestamp of the desired single bucket result
:arg body: Bucket selection details if not provided in URI
:arg anomaly_score: Filter for the most anomalous buckets
:arg desc: Set the sort direction
:arg end: End time filter for buckets
:arg exclude_interim: Exclude interim results
:arg expand: Include anomaly records
:arg from_: skips a number of buckets
:arg size: specifies a max number of buckets to get
:arg sort: Sort buckets by a particular field
:arg start: Start time filter for buckets
"""
if job_id in SKIP_IN_PATH:
raise ValueError("Empty value passed for a required argument 'job_id'.")
return self.transport.perform_request(
"GET",
_make_path(
"_xpack",
"ml",
"anomaly_detectors",
job_id,
"results",
"buckets",
timestamp,
),
params=params,
body=body,
)
@query_params("reset_end", "reset_start")
def post_data(self, job_id, body, params=None):
"""
`<http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-post-data.html>`_
:arg job_id: The name of the job receiving the data
:arg body: The data to process
:arg reset_end: Optional parameter to specify the end of the bucket
resetting range
:arg reset_start: Optional parameter to specify the start of the bucket
resetting range
"""
for param in (job_id, body):
if param in SKIP_IN_PATH:
raise ValueError("Empty value passed for a required argument.")
return self.transport.perform_request(
"POST",
_make_path("_xpack", "ml", "anomaly_detectors", job_id, "_data"),
params=params,
body=self.client._bulk_body(body),
)
@query_params("force", "timeout")
def stop_datafeed(self, datafeed_id, params=None):
"""
`<http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-stop-datafeed.html>`_
:arg datafeed_id: The ID of the datafeed to stop
:arg force: True if the datafeed should be forcefully stopped.
:arg timeout: Controls the time to wait until a datafeed has stopped.
Default to 20 seconds
"""
if datafeed_id in SKIP_IN_PATH:
raise ValueError(
"Empty value passed for a required argument 'datafeed_id'."
)
return self.transport.perform_request(
"POST",
_make_path("_xpack", "ml", "datafeeds", datafeed_id, "_stop"),
params=params,
)
@query_params()
def get_jobs(self, job_id=None, params=None):
"""
`<http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-get-job.html>`_
:arg job_id: The ID of the jobs to fetch
"""
return self.transport.perform_request(
"GET",
_make_path("_xpack", "ml", "anomaly_detectors", job_id),
params=params,
)
@query_params()
def delete_expired_data(self, params=None):
""""""
return self.transport.perform_request(
"DELETE", "/_xpack/ml/_delete_expired_data", params=params
)
@query_params()
def put_job(self, job_id, body, params=None):
"""
`<http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-put-job.html>`_
:arg job_id: The ID of the job to create
:arg body: The job
"""
for param in (job_id, body):
if param in SKIP_IN_PATH:
raise ValueError("Empty value passed for a required argument.")
return self.transport.perform_request(
"PUT",
_make_path("_xpack", "ml", "anomaly_detectors", job_id),
params=params,
body=body,
)
@query_params()
def validate_detector(self, body, params=None):
"""
:arg body: The detector
"""
if body in SKIP_IN_PATH:
raise ValueError("Empty value passed for a required argument 'body'.")
return self.transport.perform_request(
"POST",
"/_xpack/ml/anomaly_detectors/_validate/detector",
params=params,
body=body,
)
@query_params("end", "start", "timeout")
def start_datafeed(self, datafeed_id, body=None, params=None):
"""
`<http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-start-datafeed.html>`_
:arg datafeed_id: The ID of the datafeed to start
:arg body: The start datafeed parameters
:arg end: The end time when the datafeed should stop. When not set, the
datafeed continues in real time
:arg start: The start time from where the datafeed should begin
:arg timeout: Controls the time to wait until a datafeed has started.
Default to 20 seconds
"""
if datafeed_id in SKIP_IN_PATH:
raise ValueError(
"Empty value passed for a required argument 'datafeed_id'."
)
return self.transport.perform_request(
"POST",
_make_path("_xpack", "ml", "datafeeds", datafeed_id, "_start"),
params=params,
body=body,
)
@query_params(
"desc",
"end",
"exclude_interim",
"from_",
"record_score",
"size",
"sort",
"start",
)
def get_records(self, job_id, body=None, params=None):
"""
`<http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-get-record.html>`_
:arg job_id: None
:arg body: Record selection criteria
:arg desc: Set the sort direction
:arg end: End time filter for records
:arg exclude_interim: Exclude interim results
:arg from_: skips a number of records
:arg record_score:
:arg size: specifies a max number of records to get
:arg sort: Sort records by a particular field
:arg start: Start time filter for records
"""
if job_id in SKIP_IN_PATH:
raise ValueError("Empty value passed for a required argument 'job_id'.")
return self.transport.perform_request(
"GET",
_make_path(
"_xpack", "ml", "anomaly_detectors", job_id, "results", "records"
),
params=params,
body=body,
)
@query_params()
def update_job(self, job_id, body, params=None):
"""
`<http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-update-job.html>`_
:arg job_id: The ID of the job to create
:arg body: The job update settings
"""
for param in (job_id, body):
if param in SKIP_IN_PATH:
raise ValueError("Empty value passed for a required argument.")
return self.transport.perform_request(
"POST",
_make_path("_xpack", "ml", "anomaly_detectors", job_id, "_update"),
params=params,
body=body,
)
@query_params()
def put_filter(self, filter_id, body, params=None):
"""
:arg filter_id: The ID of the filter to create
:arg body: The filter details
"""
for param in (filter_id, body):
if param in SKIP_IN_PATH:
raise ValueError("Empty value passed for a required argument.")
return self.transport.perform_request(
"PUT",
_make_path("_xpack", "ml", "filters", filter_id),
params=params,
body=body,
)
@query_params()
def update_datafeed(self, datafeed_id, body, params=None):
"""
`<http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-update-datafeed.html>`_
:arg datafeed_id: The ID of the datafeed to update
:arg body: The datafeed update settings
"""
for param in (datafeed_id, body):
if param in SKIP_IN_PATH:
raise ValueError("Empty value passed for a required argument.")
return self.transport.perform_request(
"POST",
_make_path("_xpack", "ml", "datafeeds", datafeed_id, "_update"),
params=params,
body=body,
)
@query_params()
def preview_datafeed(self, datafeed_id, params=None):
"""
`<http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-preview-datafeed.html>`_
:arg datafeed_id: The ID of the datafeed to preview
"""
if datafeed_id in SKIP_IN_PATH:
raise ValueError(
"Empty value passed for a required argument 'datafeed_id'."
)
return self.transport.perform_request(
"GET",
_make_path("_xpack", "ml", "datafeeds", datafeed_id, "_preview"),
params=params,
)
@query_params("advance_time", "calc_interim", "end", "skip_time", "start")
def flush_job(self, job_id, body=None, params=None):
"""
`<http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-flush-job.html>`_
:arg job_id: The name of the job to flush
:arg body: Flush parameters
:arg advance_time: Advances time to the given value generating results
and updating the model for the advanced interval
:arg calc_interim: Calculates interim results for the most recent bucket
or all buckets within the latency period
:arg end: When used in conjunction with calc_interim, specifies the
range of buckets on which to calculate interim results
:arg skip_time: Skips time to the given value without generating results
or updating the model for the skipped interval
:arg start: When used in conjunction with calc_interim, specifies the
range of buckets on which to calculate interim results
"""
if job_id in SKIP_IN_PATH:
raise ValueError("Empty value passed for a required argument 'job_id'.")
return self.transport.perform_request(
"POST",
_make_path("_xpack", "ml", "anomaly_detectors", job_id, "_flush"),
params=params,
body=body,
)
@query_params("force", "timeout")
def close_job(self, job_id, params=None):
"""
`<http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-close-job.html>`_
:arg job_id: The name of the job to close
:arg force: True if the job should be forcefully closed
:arg timeout: Controls the time to wait until a job has closed. Default
to 30 minutes
"""
if job_id in SKIP_IN_PATH:
raise ValueError("Empty value passed for a required argument 'job_id'.")
return self.transport.perform_request(
"POST",
_make_path("_xpack", "ml", "anomaly_detectors", job_id, "_close"),
params=params,
)
@query_params()
def open_job(self, job_id, params=None):
"""
`<http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-open-job.html>`_
:arg job_id: The ID of the job to open
"""
if job_id in SKIP_IN_PATH:
raise ValueError("Empty value passed for a required argument 'job_id'.")
return self.transport.perform_request(
"POST",
_make_path("_xpack", "ml", "anomaly_detectors", job_id, "_open"),
params=params,
)
@query_params("force")
def delete_job(self, job_id, params=None):
"""
`<http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-delete-job.html>`_
:arg job_id: The ID of the job to delete
:arg force: True if the job should be forcefully deleted
"""
if job_id in SKIP_IN_PATH:
raise ValueError("Empty value passed for a required argument 'job_id'.")
return self.transport.perform_request(
"DELETE",
_make_path("_xpack", "ml", "anomaly_detectors", job_id),
params=params,
)
@query_params("duration", "expires_in")
def forecast_job(self, job_id, params=None):
"""
`<http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-forecast.html>`_
:arg job_id: The name of the job to close
:arg duration: A period of time that indicates how far into the future to forecast
:arg expires_in: The period of time that forecast results are retained.
"""
if job_id in SKIP_IN_PATH:
raise ValueError("Empty value passed for a required argument 'job_id'.")
return self.transport.perform_request(
"POST",
_make_path("_xpack", "ml", "anomaly_detectors", job_id, "_forecast"),
params=params,
)
@query_params()
def update_model_snapshot(self, job_id, snapshot_id, body, params=None):
"""
`<http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-update-snapshot.html>`_
:arg job_id: The ID of the job to fetch
:arg snapshot_id: The ID of the snapshot to update
:arg body: The model snapshot properties to update
"""
for param in (job_id, snapshot_id, body):
if param in SKIP_IN_PATH:
raise ValueError("Empty value passed for a required argument.")
return self.transport.perform_request(
"POST",
_make_path(
"_xpack",
"ml",
"anomaly_detectors",
job_id,
"model_snapshots",
snapshot_id,
"_update",
),
params=params,
body=body,
)
@query_params()
def delete_filter(self, filter_id, params=None):
"""
:arg filter_id: The ID of the filter to delete
"""
if filter_id in SKIP_IN_PATH:
raise ValueError("Empty value passed for a required argument 'filter_id'.")
return self.transport.perform_request(
"DELETE", _make_path("_xpack", "ml", "filters", filter_id), params=params
)
@query_params()
def validate(self, body, params=None):
"""
:arg body: The job config
"""
if body in SKIP_IN_PATH:
raise ValueError("Empty value passed for a required argument 'body'.")
return self.transport.perform_request(
"POST", "/_xpack/ml/anomaly_detectors/_validate", params=params, body=body
)
@query_params("from_", "size")
def get_categories(self, job_id, category_id=None, body=None, params=None):
"""
`<http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-get-category.html>`_
:arg job_id: The name of the job
:arg category_id: The identifier of the category definition of interest
:arg body: Category selection details if not provided in URI
:arg from_: skips a number of categories
:arg size: specifies a max number of categories to get
"""
if job_id in SKIP_IN_PATH:
raise ValueError("Empty value passed for a required argument 'job_id'.")
return self.transport.perform_request(
"GET",
_make_path(
"_xpack",
"ml",
"anomaly_detectors",
job_id,
"results",
"categories",
category_id,
),
params=params,
body=body,
)
@query_params(
"desc",
"end",
"exclude_interim",
"from_",
"influencer_score",
"size",
"sort",
"start",
)
def get_influencers(self, job_id, body=None, params=None):
"""
`<http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-get-influencer.html>`_
:arg job_id: None
:arg body: Influencer selection criteria
:arg desc: whether the results should be sorted in decending order
:arg end: end timestamp for the requested influencers
:arg exclude_interim: Exclude interim results
:arg from_: skips a number of influencers
:arg influencer_score: influencer score threshold for the requested
influencers
:arg size: specifies a max number of influencers to get
:arg sort: sort field for the requested influencers
:arg start: start timestamp for the requested influencers
"""
if job_id in SKIP_IN_PATH:
raise ValueError("Empty value passed for a required argument 'job_id'.")
return self.transport.perform_request(
"GET",
_make_path(
"_xpack", "ml", "anomaly_detectors", job_id, "results", "influencers"
),
params=params,
body=body,
)
@query_params()
def put_datafeed(self, datafeed_id, body, params=None):
"""
`<http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-put-datafeed.html>`_
:arg datafeed_id: The ID of the datafeed to create
:arg body: The datafeed config
"""
for param in (datafeed_id, body):
if param in SKIP_IN_PATH:
raise ValueError("Empty value passed for a required argument.")
return self.transport.perform_request(
"PUT",
_make_path("_xpack", "ml", "datafeeds", datafeed_id),
params=params,
body=body,
)
@query_params("force")
def delete_datafeed(self, datafeed_id, params=None):
"""
`<http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-delete-datafeed.html>`_
:arg datafeed_id: The ID of the datafeed to delete
:arg force: True if the datafeed should be forcefully deleted
"""
if datafeed_id in SKIP_IN_PATH:
raise ValueError(
"Empty value passed for a required argument 'datafeed_id'."
)
return self.transport.perform_request(
"DELETE",
_make_path("_xpack", "ml", "datafeeds", datafeed_id),
params=params,
)
@query_params()
def get_job_stats(self, job_id=None, params=None):
"""
`<http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-get-job-stats.html>`_
:arg job_id: The ID of the jobs stats to fetch
"""
return self.transport.perform_request(
"GET",
_make_path("_xpack", "ml", "anomaly_detectors", job_id, "_stats"),
params=params,
)
@query_params("delete_intervening_results")
def revert_model_snapshot(self, job_id, snapshot_id, body=None, params=None):
"""
`<http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-revert-snapshot.html>`_
:arg job_id: The ID of the job to fetch
:arg snapshot_id: The ID of the snapshot to revert to
:arg body: Reversion options
:arg delete_intervening_results: Should we reset the results back to the
time of the snapshot?
"""
for param in (job_id, snapshot_id):
if param in SKIP_IN_PATH:
raise ValueError("Empty value passed for a required argument.")
return self.transport.perform_request(
"POST",
_make_path(
"_xpack",
"ml",
"anomaly_detectors",
job_id,
"model_snapshots",
snapshot_id,
"_revert",
),
params=params,
body=body,
)
@query_params("desc", "end", "from_", "size", "sort", "start")
def get_model_snapshots(self, job_id, snapshot_id=None, body=None, params=None):
"""
`<http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-get-snapshot.html>`_
:arg job_id: The ID of the job to fetch
:arg snapshot_id: The ID of the snapshot to fetch
:arg body: Model snapshot selection criteria
:arg desc: True if the results should be sorted in descending order
:arg end: The filter 'end' query parameter
:arg from_: Skips a number of documents
:arg size: The default number of documents returned in queries as a
string.
:arg sort: Name of the field to sort on
:arg start: The filter 'start' query parameter
"""
if job_id in SKIP_IN_PATH:
raise ValueError("Empty value passed for a required argument 'job_id'.")
return self.transport.perform_request(
"GET",
_make_path(
"_xpack",
"ml",
"anomaly_detectors",
job_id,
"model_snapshots",
snapshot_id,
),
params=params,
body=body,
)
@query_params()
def delete_model_snapshot(self, job_id, snapshot_id, params=None):
"""
`<http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-delete-snapshot.html>`_
:arg job_id: The ID of the job to fetch
:arg snapshot_id: The ID of the snapshot to delete
"""
for param in (job_id, snapshot_id):
if param in SKIP_IN_PATH:
raise ValueError("Empty value passed for a required argument.")
return self.transport.perform_request(
"DELETE",
_make_path(
"_xpack",
"ml",
"anomaly_detectors",
job_id,
"model_snapshots",
snapshot_id,
),
params=params,
)

47
infrastructure/sandbox/Data/lambda/elasticsearch/client/xpack/monitoring.py
View file

@ -1,47 +0,0 @@
# Licensed to Elasticsearch B.V. under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch B.V. licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
from ..utils import (
NamespacedClient,
query_params,
_make_path,
SKIP_IN_PATH,
)
class MonitoringClient(NamespacedClient):
@query_params("interval", "system_api_version", "system_id")
def bulk(self, body, doc_type=None, params=None):
"""
`<http://www.elastic.co/guide/en/monitoring/current/appendix-api-bulk.html>`_
:arg body: The operation definition and data (action-data pairs),
separated by newlines
:arg doc_type: Default document type for items which don't provide one
:arg interval: Collection interval (e.g., '10s' or '10000ms') of the
payload
:arg system_api_version: API Version of the monitored system
:arg system_id: Identifier of the monitored system
"""
if body in SKIP_IN_PATH:
raise ValueError("Empty value passed for a required argument 'body'.")
return self.transport.perform_request(
"POST",
_make_path("_xpack", "monitoring", doc_type, "_bulk"),
params=params,
body=self.client._bulk_body(body),
)

318
infrastructure/sandbox/Data/lambda/elasticsearch/client/xpack/security.py
View file

@ -1,318 +0,0 @@
# Licensed to Elasticsearch B.V. under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch B.V. licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
from ..utils import (
NamespacedClient,
query_params,
_make_path,
SKIP_IN_PATH,
)
class SecurityClient(NamespacedClient):
@query_params("refresh")
def delete_user(self, username, params=None):
"""
`<https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-users.html#security-api-delete-user>`_
:arg username: username
:arg refresh: If `true` (the default) then refresh the affected shards
to make this operation visible to search, if `wait_for` then wait
for a refresh to make this operation visible to search, if `false`
then do nothing with refreshes., valid choices are: 'true', 'false',
'wait_for'
"""
if username in SKIP_IN_PATH:
raise ValueError("Empty value passed for a required argument 'username'.")
return self.transport.perform_request(
"DELETE", _make_path("_xpack", "security", "user", username), params=params
)
@query_params()
def get_user(self, username=None, params=None):
"""
`<https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-users.html#security-api-get-user>`_
:arg username: A comma-separated list of usernames
"""
return self.transport.perform_request(
"GET", _make_path("_xpack", "security", "user", username), params=params
)
@query_params("refresh")
def put_role(self, name, body, params=None):
"""
`<https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-roles.html#security-api-put-role>`_
:arg name: Role name
:arg body: The role to add
:arg refresh: If `true` (the default) then refresh the affected shards
to make this operation visible to search, if `wait_for` then wait
for a refresh to make this operation visible to search, if `false`
then do nothing with refreshes., valid choices are: 'true', 'false',
'wait_for'
"""
for param in (name, body):
if param in SKIP_IN_PATH:
raise ValueError("Empty value passed for a required argument.")
return self.transport.perform_request(
"PUT",
_make_path("_xpack", "security", "role", name),
params=params,
body=body,
)
@query_params()
def authenticate(self, params=None):
"""
`<https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-authenticate.html>`_
"""
return self.transport.perform_request(
"GET", "/_xpack/security/_authenticate", params=params
)
@query_params("refresh")
def put_user(self, username, body, params=None):
"""
`<https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-users.html#security-api-put-user>`_
:arg username: The username of the User
:arg body: The user to add
:arg refresh: If `true` (the default) then refresh the affected shards
to make this operation visible to search, if `wait_for` then wait
for a refresh to make this operation visible to search, if `false`
then do nothing with refreshes., valid choices are: 'true', 'false',
'wait_for'
"""
for param in (username, body):
if param in SKIP_IN_PATH:
raise ValueError("Empty value passed for a required argument.")
return self.transport.perform_request(
"PUT",
_make_path("_xpack", "security", "user", username),
params=params,
body=body,
)
@query_params("usernames")
def clear_cached_realms(self, realms, params=None):
"""
`<https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-clear-cache.html>`_
:arg realms: Comma-separated list of realms to clear
:arg usernames: Comma-separated list of usernames to clear from the
cache
"""
if realms in SKIP_IN_PATH:
raise ValueError("Empty value passed for a required argument 'realms'.")
return self.transport.perform_request(
"POST",
_make_path("_xpack", "security", "realm", realms, "_clear_cache"),
params=params,
)
@query_params("refresh")
def change_password(self, body, username=None, params=None):
"""
`<https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-change-password.html>`_
:arg body: the new password for the user
:arg username: The username of the user to change the password for
:arg refresh: If `true` (the default) then refresh the affected shards
to make this operation visible to search, if `wait_for` then wait
for a refresh to make this operation visible to search, if `false`
then do nothing with refreshes., valid choices are: 'true', 'false',
'wait_for'
"""
if body in SKIP_IN_PATH:
raise ValueError("Empty value passed for a required argument 'body'.")
return self.transport.perform_request(
"PUT",
_make_path("_xpack", "security", "user", username, "_password"),
params=params,
body=body,
)
@query_params()
def get_role(self, name=None, params=None):
"""
`<https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-roles.html#security-api-get-role>`_
:arg name: Role name
"""
return self.transport.perform_request(
"GET", _make_path("_xpack", "security", "role", name), params=params
)
@query_params()
def clear_cached_roles(self, name, params=None):
"""
`<https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-roles.html#security-api-clear-role-cache>`_
:arg name: Role name
"""
if name in SKIP_IN_PATH:
raise ValueError("Empty value passed for a required argument 'name'.")
return self.transport.perform_request(
"POST",
_make_path("_xpack", "security", "role", name, "_clear_cache"),
params=params,
)
@query_params("refresh")
def delete_role(self, name, params=None):
"""
`<https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-roles.html#security-api-delete-role>`_
:arg name: Role name
:arg refresh: If `true` (the default) then refresh the affected shards
to make this operation visible to search, if `wait_for` then wait
for a refresh to make this operation visible to search, if `false`
then do nothing with refreshes., valid choices are: 'true', 'false',
'wait_for'
"""
if name in SKIP_IN_PATH:
raise ValueError("Empty value passed for a required argument 'name'.")
return self.transport.perform_request(
"DELETE", _make_path("_xpack", "security", "role", name), params=params
)
@query_params("refresh")
def delete_role_mapping(self, name, params=None):
"""
`<https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-role-mapping.html#security-api-delete-role-mapping>`_
:arg name: Role-mapping name
:arg refresh: If `true` (the default) then refresh the affected shards
to make this operation visible to search, if `wait_for` then wait
for a refresh to make this operation visible to search, if `false`
then do nothing with refreshes., valid choices are: 'true', 'false',
'wait_for'
"""
if name in SKIP_IN_PATH:
raise ValueError("Empty value passed for a required argument 'name'.")
return self.transport.perform_request(
"DELETE",
_make_path("_xpack", "security", "role_mapping", name),
params=params,
)
@query_params("refresh")
def disable_user(self, username=None, params=None):
"""
`<https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-users.html#security-api-disable-user>`_
:arg username: The username of the user to disable
:arg refresh: If `true` (the default) then refresh the affected shards
to make this operation visible to search, if `wait_for` then wait
for a refresh to make this operation visible to search, if `false`
then do nothing with refreshes., valid choices are: 'true', 'false',
'wait_for'
"""
return self.transport.perform_request(
"PUT",
_make_path("_xpack", "security", "user", username, "_disable"),
params=params,
)
@query_params("refresh")
def enable_user(self, username=None, params=None):
"""
`<https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-users.html#security-api-enable-user>`_
:arg username: The username of the user to enable
:arg refresh: If `true` (the default) then refresh the affected shards
to make this operation visible to search, if `wait_for` then wait
for a refresh to make this operation visible to search, if `false`
then do nothing with refreshes., valid choices are: 'true', 'false',
'wait_for'
"""
return self.transport.perform_request(
"PUT",
_make_path("_xpack", "security", "user", username, "_enable"),
params=params,
)
@query_params()
def get_role_mapping(self, name=None, params=None):
"""
`<https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-role-mapping.html#security-api-get-role-mapping>`_
:arg name: Role-Mapping name
"""
return self.transport.perform_request(
"GET", _make_path("_xpack", "security", "role_mapping", name), params=params
)
@query_params()
def get_token(self, body, params=None):
"""
`<https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-tokens.html#security-api-get-token>`_
:arg body: The token request to get
"""
if body in SKIP_IN_PATH:
raise ValueError("Empty value passed for a required argument 'body'.")
return self.transport.perform_request(
"POST", "/_xpack/security/oauth2/token", params=params, body=body
)
@query_params()
def invalidate_token(self, body, params=None):
"""
`<https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-tokens.html#security-api-invalidate-token>`_
:arg body: The token to invalidate
"""
if body in SKIP_IN_PATH:
raise ValueError("Empty value passed for a required argument 'body'.")
return self.transport.perform_request(
"DELETE", "/_xpack/security/oauth2/token", params=params, body=body
)
@query_params("refresh")
def put_role_mapping(self, name, body, params=None):
"""
`<https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-role-mapping.html#security-api-put-role-mapping>`_
:arg name: Role-mapping name
:arg body: The role to add
:arg refresh: If `true` (the default) then refresh the affected shards
to make this operation visible to search, if `wait_for` then wait
for a refresh to make this operation visible to search, if `false`
then do nothing with refreshes., valid choices are: 'true', 'false',
'wait_for'
"""
for param in (name, body):
if param in SKIP_IN_PATH:
raise ValueError("Empty value passed for a required argument.")
return self.transport.perform_request(
"PUT",
_make_path("_xpack", "security", "role_mapping", name),
params=params,
body=body,
)

193
infrastructure/sandbox/Data/lambda/elasticsearch/client/xpack/watcher.py
View file

@ -1,193 +0,0 @@
# Licensed to Elasticsearch B.V. under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch B.V. licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
from ..utils import (
NamespacedClient,
query_params,
_make_path,
SKIP_IN_PATH,
)
class WatcherClient(NamespacedClient):
@query_params()
def stop(self, params=None):
"""
`<http://www.elastic.co/guide/en/elasticsearch/reference/current/watcher-api-stop.html>`_
"""
return self.transport.perform_request(
"POST", "/_xpack/watcher/_stop", params=params
)
@query_params("master_timeout")
def ack_watch(self, watch_id, action_id=None, params=None):
"""
`<http://www.elastic.co/guide/en/elasticsearch/reference/current/watcher-api-ack-watch.html>`_
:arg watch_id: Watch ID
:arg action_id: A comma-separated list of the action ids to be acked
:arg master_timeout: Explicit operation timeout for connection to master
node
"""
if watch_id in SKIP_IN_PATH:
raise ValueError("Empty value passed for a required argument 'watch_id'.")
return self.transport.perform_request(
"PUT",
_make_path("_xpack", "watcher", "watch", watch_id, "_ack", action_id),
params=params,
)
@query_params("debug")
def execute_watch(self, id=None, body=None, params=None):
"""
`<http://www.elastic.co/guide/en/elasticsearch/reference/current/watcher-api-execute-watch.html>`_
:arg id: Watch ID
:arg body: Execution control
:arg debug: indicates whether the watch should execute in debug mode
"""
return self.transport.perform_request(
"PUT",
_make_path("_xpack", "watcher", "watch", id, "_execute"),
params=params,
body=body,
)
@query_params()
def start(self, params=None):
"""
`<http://www.elastic.co/guide/en/elasticsearch/reference/current/watcher-api-start.html>`_
"""
return self.transport.perform_request(
"POST", "/_xpack/watcher/_start", params=params
)
@query_params("master_timeout")
def activate_watch(self, watch_id, params=None):
"""
`<https://www.elastic.co/guide/en/elasticsearch/reference/current/watcher-api-activate-watch.html>`_
:arg watch_id: Watch ID
:arg master_timeout: Explicit operation timeout for connection to master
node
"""
if watch_id in SKIP_IN_PATH:
raise ValueError("Empty value passed for a required argument 'watch_id'.")
return self.transport.perform_request(
"PUT",
_make_path("_xpack", "watcher", "watch", watch_id, "_activate"),
params=params,
)
@query_params("master_timeout")
def deactivate_watch(self, watch_id, params=None):
"""
`<https://www.elastic.co/guide/en/elasticsearch/reference/current/watcher-api-deactivate-watch.html>`_
:arg watch_id: Watch ID
:arg master_timeout: Explicit operation timeout for connection to master
node
"""
if watch_id in SKIP_IN_PATH:
raise ValueError("Empty value passed for a required argument 'watch_id'.")
return self.transport.perform_request(
"PUT",
_make_path("_xpack", "watcher", "watch", watch_id, "_deactivate"),
params=params,
)
@query_params("active", "master_timeout")
def put_watch(self, id, body, params=None):
"""
`<http://www.elastic.co/guide/en/elasticsearch/reference/current/watcher-api-put-watch.html>`_
:arg id: Watch ID
:arg body: The watch
:arg active: Specify whether the watch is in/active by default
:arg master_timeout: Explicit operation timeout for connection to master
node
"""
for param in (id, body):
if param in SKIP_IN_PATH:
raise ValueError("Empty value passed for a required argument.")
return self.transport.perform_request(
"PUT",
_make_path("_xpack", "watcher", "watch", id),
params=params,
body=body,
)
@query_params("master_timeout")
def delete_watch(self, id, params=None):
"""
`<http://www.elastic.co/guide/en/elasticsearch/reference/current/watcher-api-delete-watch.html>`_
:arg id: Watch ID
:arg master_timeout: Explicit operation timeout for connection to master
node
"""
if id in SKIP_IN_PATH:
raise ValueError("Empty value passed for a required argument 'id'.")
return self.transport.perform_request(
"DELETE", _make_path("_xpack", "watcher", "watch", id), params=params
)
@query_params()
def get_watch(self, id, params=None):
"""
`<http://www.elastic.co/guide/en/elasticsearch/reference/current/watcher-api-get-watch.html>`_
:arg id: Watch ID
"""
if id in SKIP_IN_PATH:
raise ValueError("Empty value passed for a required argument 'id'.")
return self.transport.perform_request(
"GET", _make_path("_xpack", "watcher", "watch", id), params=params
)
@query_params("emit_stacktraces")
def stats(self, metric=None, params=None):
"""
`<http://www.elastic.co/guide/en/elasticsearch/reference/current/watcher-api-stats.html>`_
:arg metric: Controls what additional stat metrics should be include in
the response
:arg emit_stacktraces: Emits stack traces of currently running watches
"""
return self.transport.perform_request(
"GET", _make_path("_xpack", "watcher", "stats", metric), params=params
)
@query_params()
def restart(self, params=None):
"""
`<http://www.elastic.co/guide/en/elasticsearch/reference/current/watcher-api-restart.html>`_
"""
return self.transport.perform_request(
"POST", "/_xpack/watcher/_restart", params=params
)

44
infrastructure/sandbox/Data/lambda/elasticsearch/compat.py
View file

@ -1,44 +0,0 @@
# Licensed to Elasticsearch B.V. under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch B.V. licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
import sys
PY2 = sys.version_info[0] == 2
if PY2:
string_types = (basestring,) # noqa: F821
from urllib import quote, quote_plus, urlencode, unquote
from urlparse import urlparse
from itertools import imap as map
from Queue import Queue
else:
string_types = str, bytes
from urllib.parse import quote, quote_plus, urlencode, urlparse, unquote
map = map
from queue import Queue
__all__ = [
"string_types",
"quote",
"quote_plus",
"urlencode",
"unquote",
"urlparse",
"map",
"Queue",
]

27
infrastructure/sandbox/Data/lambda/elasticsearch/connection/__init__.py
View file

@ -1,27 +0,0 @@
# Licensed to Elasticsearch B.V. under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch B.V. licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
from .base import Connection
from .http_requests import RequestsHttpConnection
from .http_urllib3 import Urllib3HttpConnection, create_ssl_context
__all__ = [
"Connection",
"RequestsHttpConnection",
"Urllib3HttpConnection",
"create_ssl_context",
]

BIN
infrastructure/sandbox/Data/lambda/elasticsearch/connection/__pycache__/__init__.cpython-310.pyc
View file

Binary file not shown.

BIN
infrastructure/sandbox/Data/lambda/elasticsearch/connection/__pycache__/base.cpython-310.pyc
View file

Binary file not shown.

BIN
infrastructure/sandbox/Data/lambda/elasticsearch/connection/__pycache__/http_requests.cpython-310.pyc
View file

Binary file not shown.

BIN
infrastructure/sandbox/Data/lambda/elasticsearch/connection/__pycache__/http_urllib3.cpython-310.pyc
View file

Binary file not shown.

BIN
infrastructure/sandbox/Data/lambda/elasticsearch/connection/__pycache__/pooling.cpython-310.pyc
View file

Binary file not shown.

258
infrastructure/sandbox/Data/lambda/elasticsearch/connection/base.py
View file

@ -1,258 +0,0 @@
# Licensed to Elasticsearch B.V. under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch B.V. licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
import logging
import binascii
import gzip
import io
from platform import python_version
try:
import simplejson as json
except ImportError:
import json
from ..exceptions import TransportError, ImproperlyConfigured, HTTP_EXCEPTIONS
from .. import __versionstr__
logger = logging.getLogger("elasticsearch")
# create the elasticsearch.trace logger, but only set propagate to False if the
# logger hasn't already been configured
_tracer_already_configured = "elasticsearch.trace" in logging.Logger.manager.loggerDict
tracer = logging.getLogger("elasticsearch.trace")
if not _tracer_already_configured:
tracer.propagate = False
class Connection(object):
"""
Class responsible for maintaining a connection to an Elasticsearch node. It
holds persistent connection pool to it and it's main interface
(`perform_request`) is thread-safe.
Also responsible for logging.
:arg host: hostname of the node (default: localhost)
:arg port: port to use (integer, default: 9200)
:arg use_ssl: use ssl for the connection if `True`
:arg url_prefix: optional url prefix for elasticsearch
:arg timeout: default timeout in seconds (float, default: 10)
:arg http_compress: Use gzip compression
:arg cloud_id: The Cloud ID from ElasticCloud. Convenient way to connect to cloud instances.
"""
HTTP_CLIENT_META = None
def __init__(
self,
host="localhost",
port=None,
use_ssl=False,
url_prefix="",
timeout=10,
headers=None,
http_compress=None,
cloud_id=None,
meta_header=True,
**kwargs
):
if cloud_id:
try:
_, cloud_id = cloud_id.split(":")
parent_dn, es_uuid = (
binascii.a2b_base64(cloud_id.encode("utf-8"))
.decode("utf-8")
.split("$")[:2]
)
if ":" in parent_dn:
parent_dn, _, parent_port = parent_dn.rpartition(":")
if port is None and parent_port != "443":
port = int(parent_port)
except (ValueError, IndexError):
raise ImproperlyConfigured("'cloud_id' is not properly formatted")
host = "%s.%s" % (es_uuid, parent_dn)
use_ssl = True
if http_compress is None:
http_compress = True
# If cloud_id isn't set and port is default then use 9200.
# Cloud should use '443' by default via the 'https' scheme.
elif port is None:
port = 9200
# Work-around if the implementing class doesn't
# define the headers property before calling super().__init__()
if not hasattr(self, "headers"):
self.headers = {}
headers = headers or {}
for key in headers:
self.headers[key.lower()] = headers[key]
self.headers.setdefault("content-type", "application/json")
self.headers.setdefault("user-agent", self._get_default_user_agent())
if http_compress:
self.headers["accept-encoding"] = "gzip,deflate"
scheme = kwargs.get("scheme", "http")
if use_ssl or scheme == "https":
scheme = "https"
use_ssl = True
self.use_ssl = use_ssl
self.http_compress = http_compress or False
self.hostname = host
self.port = port
self.host = "%s://%s" % (scheme, host)
if self.port is not None:
self.host += ":%s" % self.port
if url_prefix:
url_prefix = "/" + url_prefix.strip("/")
self.url_prefix = url_prefix
self.timeout = timeout
if not isinstance(meta_header, bool):
raise TypeError("meta_header must be of type bool")
self.meta_header = meta_header
def __repr__(self):
return "<%s: %s>" % (self.__class__.__name__, self.host)
def _gzip_compress(self, body):
buf = io.BytesIO()
with gzip.GzipFile(fileobj=buf, mode="wb") as f:
f.write(body)
return buf.getvalue()
def _pretty_json(self, data):
# pretty JSON in tracer curl logs
try:
return json.dumps(
json.loads(data), sort_keys=True, indent=2, separators=(",", ": ")
).replace("'", r"\u0027")
except (ValueError, TypeError):
# non-json data or a bulk request
return data
def _log_trace(self, method, path, body, status_code, response, duration):
if not tracer.isEnabledFor(logging.INFO) or not tracer.handlers:
return
# include pretty in trace curls
path = path.replace("?", "?pretty&", 1) if "?" in path else path + "?pretty"
if self.url_prefix:
path = path.replace(self.url_prefix, "", 1)
tracer.info(
"curl %s-X%s 'http://localhost:9200%s' -d '%s'",
"-H 'Content-Type: application/json' " if body else "",
method,
path,
self._pretty_json(body) if body else "",
)
if tracer.isEnabledFor(logging.DEBUG):
tracer.debug(
"#[%s] (%.3fs)\n#%s",
status_code,
duration,
self._pretty_json(response).replace("\n", "\n#") if response else "",
)
def log_request_success(
self, method, full_url, path, body, status_code, response, duration
):
""" Log a successful API call. """
# TODO: optionally pass in params instead of full_url and do urlencode only when needed
# body has already been serialized to utf-8, deserialize it for logging
# TODO: find a better way to avoid (de)encoding the body back and forth
if body:
try:
body = body.decode("utf-8", "ignore")
except AttributeError:
pass
logger.info(
"%s %s [status:%s request:%.3fs]", method, full_url, status_code, duration
)
logger.debug("> %s", body)
logger.debug("< %s", response)
self._log_trace(method, path, body, status_code, response, duration)
def log_request_fail(
self,
method,
full_url,
path,
body,
duration,
status_code=None,
response=None,
exception=None,
):
""" Log an unsuccessful API call. """
# do not log 404s on HEAD requests
if method == "HEAD" and status_code == 404:
return
logger.warning(
"%s %s [status:%s request:%.3fs]",
method,
full_url,
status_code or "N/A",
duration,
exc_info=exception is not None,
)
# body has already been serialized to utf-8, deserialize it for logging
# TODO: find a better way to avoid (de)encoding the body back and forth
if body:
try:
body = body.decode("utf-8", "ignore")
except AttributeError:
pass
logger.debug("> %s", body)
self._log_trace(method, path, body, status_code, response, duration)
if response is not None:
logger.debug("< %s", response)
def _raise_error(self, status_code, raw_data):
""" Locate appropriate exception and raise it. """
error_message = raw_data
additional_info = None
try:
if raw_data:
additional_info = json.loads(raw_data)
error_message = additional_info.get("error", error_message)
if isinstance(error_message, dict) and "type" in error_message:
error_message = error_message["type"]
except (ValueError, TypeError) as err:
logger.warning("Undecodable raw error response from server: %s", err)
raise HTTP_EXCEPTIONS.get(status_code, TransportError)(
status_code, error_message, additional_info
)
def _get_default_user_agent(self):
return "elasticsearch-py/%s (Python %s)" % (__versionstr__, python_version())

208
infrastructure/sandbox/Data/lambda/elasticsearch/connection/http_requests.py
View file

@ -1,208 +0,0 @@
# Licensed to Elasticsearch B.V. under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch B.V. licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
import time
import warnings
from .base import Connection
from ..exceptions import (
ConnectionError,
ImproperlyConfigured,
ConnectionTimeout,
SSLError,
)
from ..compat import urlencode, string_types
from ..utils import _client_meta_version
try:
import requests
REQUESTS_AVAILABLE = True
_REQUESTS_META_VERSION = _client_meta_version(requests.__version__)
except ImportError:
REQUESTS_AVAILABLE = False
_REQUESTS_META_VERSION = ""
class RequestsHttpConnection(Connection):
"""
Connection using the `requests` library.
:arg http_auth: optional http auth information as either ':' separated
string or a tuple. Any value will be passed into requests as `auth`.
:arg use_ssl: use ssl for the connection if `True`
:arg verify_certs: whether to verify SSL certificates
:arg ca_certs: optional path to CA bundle. By default standard requests'
bundle will be used.
:arg client_cert: path to the file containing the private key and the
certificate, or cert only if using client_key
:arg client_key: path to the file containing the private key if using
separate cert and key files (client_cert will contain only the cert)
:arg headers: any custom http headers to be add to requests
:arg http_compress: Use gzip compression
:arg cloud_id: The Cloud ID from ElasticCloud. Convenient way to connect to cloud instances.
Other host connection params will be ignored.
"""
HTTP_CLIENT_META = ("rq", _REQUESTS_META_VERSION)
def __init__(
self,
host="localhost",
port=None,
http_auth=None,
use_ssl=False,
verify_certs=True,
ca_certs=None,
client_cert=None,
client_key=None,
headers=None,
http_compress=None,
cloud_id=None,
**kwargs
):
if not REQUESTS_AVAILABLE:
raise ImproperlyConfigured(
"Please install requests to use RequestsHttpConnection."
)
# Initialize Session so .headers works before calling super().__init__().
self.session = requests.Session()
for key in list(self.session.headers):
self.session.headers.pop(key)
super(RequestsHttpConnection, self).__init__(
host=host,
port=port,
use_ssl=use_ssl,
headers=headers,
http_compress=http_compress,
cloud_id=cloud_id,
**kwargs
)
if not self.http_compress:
# Need to set this to 'None' otherwise Requests adds its own.
self.session.headers["accept-encoding"] = None
if http_auth is not None:
if isinstance(http_auth, (tuple, list)):
http_auth = tuple(http_auth)
elif isinstance(http_auth, string_types):
http_auth = tuple(http_auth.split(":", 1))
self.session.auth = http_auth
self.base_url = "%s%s" % (
self.host,
self.url_prefix,
)
self.session.verify = verify_certs
if not client_key:
self.session.cert = client_cert
elif client_cert:
# cert is a tuple of (certfile, keyfile)
self.session.cert = (client_cert, client_key)
if ca_certs:
if not verify_certs:
raise ImproperlyConfigured(
"You cannot pass CA certificates when verify SSL is off."
)
self.session.verify = ca_certs
if self.use_ssl and not verify_certs:
warnings.warn(
"Connecting to %s using SSL with verify_certs=False is insecure."
% self.host
)
def perform_request(
self, method, url, params=None, body=None, timeout=None, ignore=(), headers=None
):
url = self.base_url + url
headers = headers or {}
if params:
url = "%s?%s" % (url, urlencode(params))
orig_body = body
if self.http_compress and body:
body = self._gzip_compress(body)
headers["content-encoding"] = "gzip"
start = time.time()
request = requests.Request(method=method, headers=headers, url=url, data=body)
prepared_request = self.session.prepare_request(request)
settings = self.session.merge_environment_settings(
prepared_request.url, {}, None, None, None
)
send_kwargs = {"timeout": timeout or self.timeout}
send_kwargs.update(settings)
try:
response = self.session.send(prepared_request, **send_kwargs)
duration = time.time() - start
raw_data = response.content.decode("utf-8", "surrogatepass")
except Exception as e:
self.log_request_fail(
method,
url,
prepared_request.path_url,
body,
time.time() - start,
exception=e,
)
if isinstance(e, requests.exceptions.SSLError):
raise SSLError("N/A", str(e), e)
if isinstance(e, requests.Timeout):
raise ConnectionTimeout("TIMEOUT", str(e), e)
raise ConnectionError("N/A", str(e), e)
# raise errors based on http status codes, let the client handle those if needed
if (
not (200 <= response.status_code < 300)
and response.status_code not in ignore
):
self.log_request_fail(
method,
url,
response.request.path_url,
orig_body,
duration,
response.status_code,
raw_data,
)
self._raise_error(response.status_code, raw_data)
self.log_request_success(
method,
url,
response.request.path_url,
orig_body,
response.status_code,
raw_data,
duration,
)
return response.status_code, response.headers, raw_data
@property
def headers(self):
return self.session.headers
def close(self):
"""
Explicitly closes connections
"""
self.session.close()

264
infrastructure/sandbox/Data/lambda/elasticsearch/connection/http_urllib3.py
View file

@ -1,264 +0,0 @@
# Licensed to Elasticsearch B.V. under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch B.V. licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
import time
import ssl
import urllib3
from urllib3.exceptions import ReadTimeoutError, SSLError as UrllibSSLError
from urllib3.util.retry import Retry
import warnings
from .base import Connection
from ..exceptions import (
ConnectionError,
ImproperlyConfigured,
ConnectionTimeout,
SSLError,
)
from ..compat import urlencode
from ..utils import _client_meta_version
# sentinel value for `verify_certs`.
# This is used to detect if a user is passing in a value for `verify_certs`
# so we can raise a warning if using SSL kwargs AND SSLContext.
VERIFY_CERTS_DEFAULT = None
CA_CERTS = None
try:
import certifi
CA_CERTS = certifi.where()
except ImportError:
pass
def create_ssl_context(**kwargs):
"""
A helper function around creating an SSL context
https://docs.python.org/3/library/ssl.html#context-creation
Accepts kwargs in the same manner as `create_default_context`.
"""
ctx = ssl.create_default_context(**kwargs)
return ctx
class Urllib3HttpConnection(Connection):
"""
Default connection class using the `urllib3` library and the http protocol.
:arg host: hostname of the node (default: localhost)
:arg port: port to use (integer, default: 9200)
:arg url_prefix: optional url prefix for elasticsearch
:arg timeout: default timeout in seconds (float, default: 10)
:arg http_auth: optional http auth information as either ':' separated
string or a tuple
:arg use_ssl: use ssl for the connection if `True`
:arg verify_certs: whether to verify SSL certificates
:arg ca_certs: optional path to CA bundle.
See https://urllib3.readthedocs.io/en/latest/security.html#using-certifi-with-urllib3
for instructions how to get default set
:arg client_cert: path to the file containing the private key and the
certificate, or cert only if using client_key
:arg client_key: path to the file containing the private key if using
separate cert and key files (client_cert will contain only the cert)
:arg ssl_version: version of the SSL protocol to use. Choices are:
SSLv23 (default) SSLv2 SSLv3 TLSv1 (see ``PROTOCOL_*`` constants in the
``ssl`` module for exact options for your environment).
:arg ssl_assert_hostname: use hostname verification if not `False`
:arg ssl_assert_fingerprint: verify the supplied certificate fingerprint if not `None`
:arg maxsize: the number of connections which will be kept open to this
host. See https://urllib3.readthedocs.io/en/1.4/pools.html#api for more
information.
:arg headers: any custom http headers to be add to requests
:arg http_compress: Use gzip compression
:arg cloud_id: The Cloud ID from ElasticCloud. Convenient way to connect to cloud instances.
Other host connection params will be ignored.
"""
HTTP_CLIENT_META = ("ur", _client_meta_version(urllib3.__version__))
def __init__(
self,
host="localhost",
port=None,
http_auth=None,
use_ssl=False,
verify_certs=VERIFY_CERTS_DEFAULT,
ca_certs=None,
client_cert=None,
client_key=None,
ssl_version=None,
ssl_assert_hostname=None,
ssl_assert_fingerprint=None,
maxsize=10,
headers=None,
ssl_context=None,
http_compress=None,
cloud_id=None,
**kwargs
):
# Initialize headers before calling super().__init__().
self.headers = urllib3.make_headers(keep_alive=True)
super(Urllib3HttpConnection, self).__init__(
host=host,
port=port,
use_ssl=use_ssl,
headers=headers,
http_compress=http_compress,
cloud_id=cloud_id,
**kwargs
)
if http_auth is not None:
if isinstance(http_auth, (tuple, list)):
http_auth = ":".join(http_auth)
self.headers.update(urllib3.make_headers(basic_auth=http_auth))
pool_class = urllib3.HTTPConnectionPool
kw = {}
# if providing an SSL context, raise error if any other SSL related flag is used
if ssl_context and (
(verify_certs is not VERIFY_CERTS_DEFAULT)
or ca_certs
or client_cert
or client_key
or ssl_version
):
warnings.warn(
"When using `ssl_context`, all other SSL related kwargs are ignored"
)
# if ssl_context provided use SSL by default
if ssl_context and self.use_ssl:
pool_class = urllib3.HTTPSConnectionPool
kw.update(
{
"assert_fingerprint": ssl_assert_fingerprint,
"ssl_context": ssl_context,
}
)
elif self.use_ssl:
pool_class = urllib3.HTTPSConnectionPool
kw.update(
{
"ssl_version": ssl_version,
"assert_hostname": ssl_assert_hostname,
"assert_fingerprint": ssl_assert_fingerprint,
}
)
# If `verify_certs` is sentinal value, default `verify_certs` to `True`
if verify_certs is VERIFY_CERTS_DEFAULT:
verify_certs = True
ca_certs = CA_CERTS if ca_certs is None else ca_certs
if verify_certs:
if not ca_certs:
raise ImproperlyConfigured(
"Root certificates are missing for certificate "
"validation. Either pass them in using the ca_certs parameter or "
"install certifi to use it automatically."
)
kw.update(
{
"cert_reqs": "CERT_REQUIRED",
"ca_certs": ca_certs,
"cert_file": client_cert,
"key_file": client_key,
}
)
else:
warnings.warn(
"Connecting to %s using SSL with verify_certs=False is insecure."
% self.host
)
kw["cert_reqs"] = "CERT_NONE"
self.pool = pool_class(
self.hostname, port=self.port, timeout=self.timeout, maxsize=maxsize, **kw
)
def perform_request(
self, method, url, params=None, body=None, timeout=None, ignore=(), headers=None
):
url = self.url_prefix + url
if params:
url = "%s?%s" % (url, urlencode(params))
full_url = self.host + url
start = time.time()
orig_body = body
try:
kw = {}
if timeout:
kw["timeout"] = timeout
# in python2 we need to make sure the url and method are not
# unicode. Otherwise the body will be decoded into unicode too and
# that will fail (#133, #201).
if not isinstance(url, str):
url = url.encode("utf-8")
if not isinstance(method, str):
method = method.encode("utf-8")
request_headers = self.headers.copy()
request_headers.update(headers or ())
if self.http_compress and body:
body = self._gzip_compress(body)
request_headers["content-encoding"] = "gzip"
response = self.pool.urlopen(
method, url, body, retries=Retry(False), headers=request_headers, **kw
)
duration = time.time() - start
raw_data = response.data.decode("utf-8", "surrogatepass")
except Exception as e:
self.log_request_fail(
method, full_url, url, orig_body, time.time() - start, exception=e
)
if isinstance(e, UrllibSSLError):
raise SSLError("N/A", str(e), e)
if isinstance(e, ReadTimeoutError):
raise ConnectionTimeout("TIMEOUT", str(e), e)
raise ConnectionError("N/A", str(e), e)
# raise errors based on http status codes, let the client handle those if needed
if not (200 <= response.status < 300) and response.status not in ignore:
self.log_request_fail(
method, full_url, url, orig_body, duration, response.status, raw_data
)
self._raise_error(response.status, raw_data)
self.log_request_success(
method, full_url, url, orig_body, response.status, raw_data, duration
)
return response.status, response.getheaders(), raw_data
def close(self):
"""
Explicitly closes connection
"""
self.pool.close()

50
infrastructure/sandbox/Data/lambda/elasticsearch/connection/pooling.py
View file

@ -1,50 +0,0 @@
# Licensed to Elasticsearch B.V. under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch B.V. licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
try:
import queue
except ImportError:
import Queue as queue
from .base import Connection
class PoolingConnection(Connection):
"""
Base connection class for connections that use libraries without thread
safety and no capacity for connection pooling. To use this just implement a
``_make_connection`` method that constructs a new connection and returns
it.
"""
def __init__(self, *args, **kwargs):
self._free_connections = queue.Queue()
super(PoolingConnection, self).__init__(*args, **kwargs)
def _get_connection(self):
try:
return self._free_connections.get_nowait()
except queue.Empty:
return self._make_connection()
def _release_connection(self, con):
self._free_connections.put(con)
def close(self):
"""
Explicitly close connection
"""
pass

295
infrastructure/sandbox/Data/lambda/elasticsearch/connection_pool.py
View file

@ -1,295 +0,0 @@
# Licensed to Elasticsearch B.V. under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch B.V. licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
import time
import random
import logging
import threading
try:
from Queue import PriorityQueue, Empty
except ImportError:
from queue import PriorityQueue, Empty
from .exceptions import ImproperlyConfigured
logger = logging.getLogger("elasticsearch")
class ConnectionSelector(object):
"""
Simple class used to select a connection from a list of currently live
connection instances. In init time it is passed a dictionary containing all
the connections' options which it can then use during the selection
process. When the `select` method is called it is given a list of
*currently* live connections to choose from.
The options dictionary is the one that has been passed to
:class:`~elasticsearch.Transport` as `hosts` param and the same that is
used to construct the Connection object itself. When the Connection was
created from information retrieved from the cluster via the sniffing
process it will be the dictionary returned by the `host_info_callback`.
Example of where this would be useful is a zone-aware selector that would
only select connections from it's own zones and only fall back to other
connections where there would be none in it's zones.
"""
def __init__(self, opts):
"""
:arg opts: dictionary of connection instances and their options
"""
self.connection_opts = opts
def select(self, connections):
"""
Select a connection from the given list.
:arg connections: list of live connections to choose from
"""
pass
class RandomSelector(ConnectionSelector):
"""
Select a connection at random
"""
def select(self, connections):
return random.choice(connections)
class RoundRobinSelector(ConnectionSelector):
"""
Selector using round-robin.
"""
def __init__(self, opts):
super(RoundRobinSelector, self).__init__(opts)
self.data = threading.local()
def select(self, connections):
self.data.rr = getattr(self.data, "rr", -1) + 1
self.data.rr %= len(connections)
return connections[self.data.rr]
class ConnectionPool(object):
"""
Container holding the :class:`~elasticsearch.Connection` instances,
managing the selection process (via a
:class:`~elasticsearch.ConnectionSelector`) and dead connections.
It's only interactions are with the :class:`~elasticsearch.Transport` class
that drives all the actions within `ConnectionPool`.
Initially connections are stored on the class as a list and, along with the
connection options, get passed to the `ConnectionSelector` instance for
future reference.
Upon each request the `Transport` will ask for a `Connection` via the
`get_connection` method. If the connection fails (it's `perform_request`
raises a `ConnectionError`) it will be marked as dead (via `mark_dead`) and
put on a timeout (if it fails N times in a row the timeout is exponentially
longer - the formula is `default_timeout * 2 ** (fail_count - 1)`). When
the timeout is over the connection will be resurrected and returned to the
live pool. A connection that has been previously marked as dead and
succeeds will be marked as live (its fail count will be deleted).
"""
def __init__(
self,
connections,
dead_timeout=60,
timeout_cutoff=5,
selector_class=RoundRobinSelector,
randomize_hosts=True,
**kwargs
):
"""
:arg connections: list of tuples containing the
:class:`~elasticsearch.Connection` instance and it's options
:arg dead_timeout: number of seconds a connection should be retired for
after a failure, increases on consecutive failures
:arg timeout_cutoff: number of consecutive failures after which the
timeout doesn't increase
:arg selector_class: :class:`~elasticsearch.ConnectionSelector`
subclass to use if more than one connection is live
:arg randomize_hosts: shuffle the list of connections upon arrival to
avoid dog piling effect across processes
"""
if not connections:
raise ImproperlyConfigured(
"No defined connections, you need to " "specify at least one host."
)
self.connection_opts = connections
self.connections = [c for (c, opts) in connections]
# remember original connection list for resurrect(force=True)
self.orig_connections = tuple(self.connections)
# PriorityQueue for thread safety and ease of timeout management
self.dead = PriorityQueue(len(self.connections))
self.dead_count = {}
if randomize_hosts:
# randomize the connection list to avoid all clients hitting same node
# after startup/restart
random.shuffle(self.connections)
# default timeout after which to try resurrecting a connection
self.dead_timeout = dead_timeout
self.timeout_cutoff = timeout_cutoff
self.selector = selector_class(dict(connections))
def mark_dead(self, connection, now=None):
"""
Mark the connection as dead (failed). Remove it from the live pool and
put it on a timeout.
:arg connection: the failed instance
"""
# allow inject for testing purposes
now = now if now else time.time()
try:
self.connections.remove(connection)
except ValueError:
# connection not alive or another thread marked it already, ignore
return
else:
dead_count = self.dead_count.get(connection, 0) + 1
self.dead_count[connection] = dead_count
timeout = self.dead_timeout * 2 ** min(dead_count - 1, self.timeout_cutoff)
self.dead.put((now + timeout, connection))
logger.warning(
"Connection %r has failed for %i times in a row, putting on %i second timeout.",
connection,
dead_count,
timeout,
)
def mark_live(self, connection):
"""
Mark connection as healthy after a resurrection. Resets the fail
counter for the connection.
:arg connection: the connection to redeem
"""
try:
del self.dead_count[connection]
except KeyError:
# race condition, safe to ignore
pass
def resurrect(self, force=False):
"""
Attempt to resurrect a connection from the dead pool. It will try to
locate one (not all) eligible (it's timeout is over) connection to
return to the live pool. Any resurrected connection is also returned.
:arg force: resurrect a connection even if there is none eligible (used
when we have no live connections). If force is specified resurrect
always returns a connection.
"""
# no dead connections
if self.dead.empty():
# we are forced to return a connection, take one from the original
# list. This is to avoid a race condition where get_connection can
# see no live connections but when it calls resurrect self.dead is
# also empty. We assume that other threat has resurrected all
# available connections so we can safely return one at random.
if force:
return random.choice(self.orig_connections)
return
try:
# retrieve a connection to check
timeout, connection = self.dead.get(block=False)
except Empty:
# other thread has been faster and the queue is now empty. If we
# are forced, return a connection at random again.
if force:
return random.choice(self.orig_connections)
return
if not force and timeout > time.time():
# return it back if not eligible and not forced
self.dead.put((timeout, connection))
return
# either we were forced or the connection is elligible to be retried
self.connections.append(connection)
logger.info("Resurrecting connection %r (force=%s).", connection, force)
return connection
def get_connection(self):
"""
Return a connection from the pool using the `ConnectionSelector`
instance.
It tries to resurrect eligible connections, forces a resurrection when
no connections are availible and passes the list of live connections to
the selector instance to choose from.
Returns a connection instance and it's current fail count.
"""
self.resurrect()
connections = self.connections[:]
# no live nodes, resurrect one by force and return it
if not connections:
return self.resurrect(True)
# only call selector if we have a selection
if len(connections) > 1:
return self.selector.select(connections)
# only one connection, no need for a selector
return connections[0]
def close(self):
"""
Explicitly closes connections
"""
for conn in self.orig_connections:
conn.close()
class DummyConnectionPool(ConnectionPool):
def __init__(self, connections, **kwargs):
if len(connections) != 1:
raise ImproperlyConfigured(
"DummyConnectionPool needs exactly one " "connection defined."
)
# we need connection opts for sniffing logic
self.connection_opts = connections
self.connection = connections[0][0]
self.connections = (self.connection,)
def get_connection(self):
return self.connection
def close(self):
"""
Explicitly closes connections
"""
self.connection.close()
def _noop(self, *args, **kwargs):
pass
mark_dead = mark_live = resurrect = _noop

156
infrastructure/sandbox/Data/lambda/elasticsearch/exceptions.py
View file

@ -1,156 +0,0 @@
# Licensed to Elasticsearch B.V. under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch B.V. licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
__all__ = [
"ImproperlyConfigured",
"ElasticsearchException",
"SerializationError",
"TransportError",
"NotFoundError",
"ConflictError",
"RequestError",
"ConnectionError",
"SSLError",
"ConnectionTimeout",
"AuthenticationException",
"AuthorizationException",
]
class ImproperlyConfigured(Exception):
"""
Exception raised when the config passed to the client is inconsistent or invalid.
"""
class ElasticsearchException(Exception):
"""
Base class for all exceptions raised by this package's operations (doesn't
apply to :class:`~elasticsearch.ImproperlyConfigured`).
"""
class SerializationError(ElasticsearchException):
"""
Data passed in failed to serialize properly in the ``Serializer`` being
used.
"""
class TransportError(ElasticsearchException):
"""
Exception raised when ES returns a non-OK (>=400) HTTP status code. Or when
an actual connection error happens; in that case the ``status_code`` will
be set to ``'N/A'``.
"""
@property
def status_code(self):
"""
The HTTP status code of the response that precipitated the error or
``'N/A'`` if not applicable.
"""
return self.args[0]
@property
def error(self):
""" A string error message. """
return self.args[1]
@property
def info(self):
"""
Dict of returned error info from ES, where available, underlying
exception when not.
"""
return self.args[2]
def __str__(self):
cause = ""
try:
if self.info and "error" in self.info:
if isinstance(self.info["error"], dict):
cause = ", %r" % self.info["error"]["root_cause"][0]["reason"]
else:
cause = ", %r" % self.info["error"]
except LookupError:
pass
return "%s(%s, %r%s)" % (
self.__class__.__name__,
self.status_code,
self.error,
cause,
)
class ConnectionError(TransportError):
"""
Error raised when there was an exception while talking to ES. Original
exception from the underlying :class:`~elasticsearch.Connection`
implementation is available as ``.info.``
"""
def __str__(self):
return "ConnectionError(%s) caused by: %s(%s)" % (
self.error,
self.info.__class__.__name__,
self.info,
)
class SSLError(ConnectionError):
""" Error raised when encountering SSL errors. """
class ConnectionTimeout(ConnectionError):
""" A network timeout. Doesn't cause a node retry by default. """
def __str__(self):
return "ConnectionTimeout caused by - %s(%s)" % (
self.info.__class__.__name__,
self.info,
)
class NotFoundError(TransportError):
""" Exception representing a 404 status code. """
class ConflictError(TransportError):
""" Exception representing a 409 status code. """
class RequestError(TransportError):
""" Exception representing a 400 status code. """
class AuthenticationException(TransportError):
""" Exception representing a 401 status code. """
class AuthorizationException(TransportError):
""" Exception representing a 403 status code. """
# more generic mappings from status_code to python exceptions
HTTP_EXCEPTIONS = {
400: RequestError,
401: AuthenticationException,
403: AuthorizationException,
404: NotFoundError,
409: ConflictError,
}

34
infrastructure/sandbox/Data/lambda/elasticsearch/helpers/__init__.py
View file

@ -1,34 +0,0 @@
# Licensed to Elasticsearch B.V. under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch B.V. licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
from .errors import BulkIndexError, ScanError
from .actions import expand_action, streaming_bulk, bulk, parallel_bulk
from .actions import scan, reindex
from .actions import _chunk_actions, _process_bulk_chunk
__all__ = [
"BulkIndexError",
"ScanError",
"expand_action",
"streaming_bulk",
"bulk",
"parallel_bulk",
"scan",
"reindex",
"_chunk_actions",
"_process_bulk_chunk",
]

BIN
infrastructure/sandbox/Data/lambda/elasticsearch/helpers/__pycache__/__init__.cpython-310.pyc
View file

Binary file not shown.

BIN
infrastructure/sandbox/Data/lambda/elasticsearch/helpers/__pycache__/actions.cpython-310.pyc
View file

Binary file not shown.

BIN
infrastructure/sandbox/Data/lambda/elasticsearch/helpers/__pycache__/errors.cpython-310.pyc
View file

Binary file not shown.

BIN
infrastructure/sandbox/Data/lambda/elasticsearch/helpers/__pycache__/test.cpython-310.pyc
View file

Binary file not shown.

565
infrastructure/sandbox/Data/lambda/elasticsearch/helpers/actions.py
View file

@ -1,565 +0,0 @@
# Licensed to Elasticsearch B.V. under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch B.V. licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
from operator import methodcaller
import time
from ..exceptions import TransportError
from ..compat import map, string_types, Queue
from .errors import ScanError, BulkIndexError
import logging
logger = logging.getLogger("elasticsearch.helpers")
def expand_action(data):
"""
From one document or action definition passed in by the user extract the
action/data lines needed for elasticsearch's
:meth:`~elasticsearch.Elasticsearch.bulk` api.
"""
# when given a string, assume user wants to index raw json
if isinstance(data, string_types):
return '{"index":{}}', data
# make sure we don't alter the action
data = data.copy()
op_type = data.pop("_op_type", "index")
action = {op_type: {}}
for key in (
"_index",
"_parent",
"_percolate",
"_routing",
"_timestamp",
"routing",
"_type",
"_version",
"_version_type",
"_id",
"retry_on_conflict",
"pipeline",
):
if key in data:
action[op_type][key] = data.pop(key)
# no data payload for delete
if op_type == "delete":
return action, None
return action, data.get("_source", data)
def _chunk_actions(actions, chunk_size, max_chunk_bytes, serializer):
"""
Split actions into chunks by number or size, serialize them into strings in
the process.
"""
bulk_actions, bulk_data = [], []
size, action_count = 0, 0
for action, data in actions:
raw_data, raw_action = data, action
action = serializer.dumps(action)
# +1 to account for the trailing new line character
cur_size = len(action.encode("utf-8")) + 1
if data is not None:
data = serializer.dumps(data)
cur_size += len(data.encode("utf-8")) + 1
# full chunk, send it and start a new one
if bulk_actions and (
size + cur_size > max_chunk_bytes or action_count == chunk_size
):
yield bulk_data, bulk_actions
bulk_actions, bulk_data = [], []
size, action_count = 0, 0
bulk_actions.append(action)
if data is not None:
bulk_actions.append(data)
bulk_data.append((raw_action, raw_data))
else:
bulk_data.append((raw_action,))
size += cur_size
action_count += 1
if bulk_actions:
yield bulk_data, bulk_actions
def _process_bulk_chunk(
client,
bulk_actions,
bulk_data,
raise_on_exception=True,
raise_on_error=True,
*args,
**kwargs
):
"""
Send a bulk request to elasticsearch and process the output.
"""
# if raise on error is set, we need to collect errors per chunk before raising them
errors = []
try:
# send the actual request
resp = client.bulk("\n".join(bulk_actions) + "\n", *args, **kwargs)
except TransportError as e:
# default behavior - just propagate exception
if raise_on_exception:
raise e
# if we are not propagating, mark all actions in current chunk as failed
err_message = str(e)
exc_errors = []
for data in bulk_data:
# collect all the information about failed actions
op_type, action = data[0].copy().popitem()
info = {"error": err_message, "status": e.status_code, "exception": e}
if op_type != "delete":
info["data"] = data[1]
info.update(action)
exc_errors.append({op_type: info})
# emulate standard behavior for failed actions
if raise_on_error:
raise BulkIndexError(
"%i document(s) failed to index." % len(exc_errors), exc_errors
)
else:
for err in exc_errors:
yield False, err
return
# go through request-response pairs and detect failures
for data, (op_type, item) in zip(
bulk_data, map(methodcaller("popitem"), resp["items"])
):
ok = 200 <= item.get("status", 500) < 300
if not ok and raise_on_error:
# include original document source
if len(data) > 1:
item["data"] = data[1]
errors.append({op_type: item})
if ok or not errors:
# if we are not just recording all errors to be able to raise
# them all at once, yield items individually
yield ok, {op_type: item}
if errors:
raise BulkIndexError("%i document(s) failed to index." % len(errors), errors)
def streaming_bulk(
client,
actions,
chunk_size=500,
max_chunk_bytes=100 * 1024 * 1024,
raise_on_error=True,
expand_action_callback=expand_action,
raise_on_exception=True,
max_retries=0,
initial_backoff=2,
max_backoff=600,
yield_ok=True,
*args,
**kwargs
):
"""
Streaming bulk consumes actions from the iterable passed in and yields
results per action. For non-streaming usecases use
:func:`~elasticsearch.helpers.bulk` which is a wrapper around streaming
bulk that returns summary information about the bulk operation once the
entire input is consumed and sent.
If you specify ``max_retries`` it will also retry any documents that were
rejected with a ``429`` status code. To do this it will wait (**by calling
time.sleep which will block**) for ``initial_backoff`` seconds and then,
every subsequent rejection for the same chunk, for double the time every
time up to ``max_backoff`` seconds.
:arg client: instance of :class:`~elasticsearch.Elasticsearch` to use
:arg actions: iterable containing the actions to be executed
:arg chunk_size: number of docs in one chunk sent to es (default: 500)
:arg max_chunk_bytes: the maximum size of the request in bytes (default: 100MB)
:arg raise_on_error: raise ``BulkIndexError`` containing errors (as `.errors`)
from the execution of the last chunk when some occur. By default we raise.
:arg raise_on_exception: if ``False`` then don't propagate exceptions from
call to ``bulk`` and just report the items that failed as failed.
:arg expand_action_callback: callback executed on each action passed in,
should return a tuple containing the action line and the data line
(`None` if data line should be omitted).
:arg max_retries: maximum number of times a document will be retried when
``429`` is received, set to 0 (default) for no retries on ``429``
:arg initial_backoff: number of seconds we should wait before the first
retry. Any subsequent retries will be powers of ``initial_backoff *
2**retry_number``
:arg max_backoff: maximum number of seconds a retry will wait
:arg yield_ok: if set to False will skip successful documents in the output
"""
actions = map(expand_action_callback, actions)
for bulk_data, bulk_actions in _chunk_actions(
actions, chunk_size, max_chunk_bytes, client.transport.serializer
):
for attempt in range(max_retries + 1):
to_retry, to_retry_data = [], []
if attempt:
time.sleep(min(max_backoff, initial_backoff * 2 ** (attempt - 1)))
try:
for data, (ok, info) in zip(
bulk_data,
_process_bulk_chunk(
client,
bulk_actions,
bulk_data,
raise_on_exception,
raise_on_error,
*args,
**kwargs
),
):
if not ok:
action, info = info.popitem()
# retry if retries enabled, we get 429, and we are not
# in the last attempt
if (
max_retries
and info["status"] == 429
and (attempt + 1) <= max_retries
):
# _process_bulk_chunk expects strings so we need to
# re-serialize the data
to_retry.extend(
map(client.transport.serializer.dumps, data)
)
to_retry_data.append(data)
else:
yield ok, {action: info}
elif yield_ok:
yield ok, info
except TransportError as e:
# suppress 429 errors since we will retry them
if attempt == max_retries or e.status_code != 429:
raise
else:
if not to_retry:
break
# retry only subset of documents that didn't succeed
bulk_actions, bulk_data = to_retry, to_retry_data
def bulk(client, actions, stats_only=False, *args, **kwargs):
"""
Helper for the :meth:`~elasticsearch.Elasticsearch.bulk` api that provides
a more human friendly interface - it consumes an iterator of actions and
sends them to elasticsearch in chunks. It returns a tuple with summary
information - number of successfully executed actions and either list of
errors or number of errors if ``stats_only`` is set to ``True``. Note that
by default we raise a ``BulkIndexError`` when we encounter an error so
options like ``stats_only`` only apply when ``raise_on_error`` is set to
``False``.
When errors are being collected original document data is included in the
error dictionary which can lead to an extra high memory usage. If you need
to process a lot of data and want to ignore/collect errors please consider
using the :func:`~elasticsearch.helpers.streaming_bulk` helper which will
just return the errors and not store them in memory.
:arg client: instance of :class:`~elasticsearch.Elasticsearch` to use
:arg actions: iterator containing the actions
:arg stats_only: if `True` only report number of successful/failed
operations instead of just number of successful and a list of error responses
Any additional keyword arguments will be passed to
:func:`~elasticsearch.helpers.streaming_bulk` which is used to execute
the operation, see :func:`~elasticsearch.helpers.streaming_bulk` for more
accepted parameters.
"""
success, failed = 0, 0
# list of errors to be collected is not stats_only
errors = []
# make streaming_bulk yield successful results so we can count them
kwargs["yield_ok"] = True
for ok, item in streaming_bulk(client, actions, *args, **kwargs):
# go through request-response pairs and detect failures
if not ok:
if not stats_only:
errors.append(item)
failed += 1
else:
success += 1
return success, failed if stats_only else errors
def parallel_bulk(
client,
actions,
thread_count=4,
chunk_size=500,
max_chunk_bytes=100 * 1024 * 1024,
queue_size=4,
expand_action_callback=expand_action,
*args,
**kwargs
):
"""
Parallel version of the bulk helper run in multiple threads at once.
:arg client: instance of :class:`~elasticsearch.Elasticsearch` to use
:arg actions: iterator containing the actions
:arg thread_count: size of the threadpool to use for the bulk requests
:arg chunk_size: number of docs in one chunk sent to es (default: 500)
:arg max_chunk_bytes: the maximum size of the request in bytes (default: 100MB)
:arg raise_on_error: raise ``BulkIndexError`` containing errors (as `.errors`)
from the execution of the last chunk when some occur. By default we raise.
:arg raise_on_exception: if ``False`` then don't propagate exceptions from
call to ``bulk`` and just report the items that failed as failed.
:arg expand_action_callback: callback executed on each action passed in,
should return a tuple containing the action line and the data line
(`None` if data line should be omitted).
:arg queue_size: size of the task queue between the main thread (producing
chunks to send) and the processing threads.
"""
# Avoid importing multiprocessing unless parallel_bulk is used
# to avoid exceptions on restricted environments like App Engine
from multiprocessing.pool import ThreadPool
actions = map(expand_action_callback, actions)
class BlockingPool(ThreadPool):
def _setup_queues(self):
super(BlockingPool, self)._setup_queues()
self._inqueue = Queue(queue_size)
self._quick_put = self._inqueue.put
pool = BlockingPool(thread_count)
try:
for result in pool.imap(
lambda bulk_chunk: list(
_process_bulk_chunk(
client, bulk_chunk[1], bulk_chunk[0], *args, **kwargs
)
),
_chunk_actions(
actions, chunk_size, max_chunk_bytes, client.transport.serializer
),
):
for item in result:
yield item
finally:
pool.close()
pool.join()
def scan(
client,
query=None,
scroll="5m",
raise_on_error=True,
preserve_order=False,
size=1000,
request_timeout=None,
clear_scroll=True,
scroll_kwargs=None,
**kwargs
):
"""
Simple abstraction on top of the
:meth:`~elasticsearch.Elasticsearch.scroll` api - a simple iterator that
yields all hits as returned by underlining scroll requests.
By default scan does not return results in any pre-determined order. To
have a standard order in the returned documents (either by score or
explicit sort definition) when scrolling, use ``preserve_order=True``. This
may be an expensive operation and will negate the performance benefits of
using ``scan``.
:arg client: instance of :class:`~elasticsearch.Elasticsearch` to use
:arg query: body for the :meth:`~elasticsearch.Elasticsearch.search` api
:arg scroll: Specify how long a consistent view of the index should be
maintained for scrolled search
:arg raise_on_error: raises an exception (``ScanError``) if an error is
encountered (some shards fail to execute). By default we raise.
:arg preserve_order: don't set the ``search_type`` to ``scan`` - this will
cause the scroll to paginate with preserving the order. Note that this
can be an extremely expensive operation and can easily lead to
unpredictable results, use with caution.
:arg size: size (per shard) of the batch send at each iteration.
:arg request_timeout: explicit timeout for each call to ``scan``
:arg clear_scroll: explicitly calls delete on the scroll id via the clear
scroll API at the end of the method on completion or error, defaults
to true.
:arg scroll_kwargs: additional kwargs to be passed to
:meth:`~elasticsearch.Elasticsearch.scroll`
Any additional keyword arguments will be passed to the initial
:meth:`~elasticsearch.Elasticsearch.search` call::
scan(es,
query={"query": {"match": {"title": "python"}}},
index="orders-*",
doc_type="books"
)
"""
scroll_kwargs = scroll_kwargs or {}
_add_helper_meta_to_kwargs(scroll_kwargs, "s")
if not preserve_order:
query = query.copy() if query else {}
query["sort"] = "_doc"
# initial search
resp = client.search(
body=query, scroll=scroll, size=size, request_timeout=request_timeout, **kwargs
)
scroll_id = resp.get("_scroll_id")
if scroll_id is None:
return
try:
first_run = True
while True:
# if we didn't set search_type to scan initial search contains data
if first_run:
first_run = False
else:
resp = client.scroll(
scroll_id=scroll_id,
scroll=scroll,
request_timeout=request_timeout,
**scroll_kwargs
)
for hit in resp["hits"]["hits"]:
yield hit
# check if we have any errrors
if resp["_shards"]["successful"] < resp["_shards"]["total"]:
logger.warning(
"Scroll request has only succeeded on %d shards out of %d.",
resp["_shards"]["successful"],
resp["_shards"]["total"],
)
if raise_on_error:
raise ScanError(
scroll_id,
"Scroll request has only succeeded on %d shards out of %d."
% (resp["_shards"]["successful"], resp["_shards"]["total"]),
)
scroll_id = resp.get("_scroll_id")
# end of scroll
if scroll_id is None or not resp["hits"]["hits"]:
break
finally:
if scroll_id and clear_scroll:
client.clear_scroll(
body={"scroll_id": [scroll_id]},
ignore=(404,),
params={"__elastic_client_meta": (("h", "s"),)},
)
def reindex(
client,
source_index,
target_index,
query=None,
target_client=None,
chunk_size=500,
scroll="5m",
scan_kwargs={},
bulk_kwargs={},
):
"""
Reindex all documents from one index that satisfy a given query
to another, potentially (if `target_client` is specified) on a different cluster.
If you don't specify the query you will reindex all the documents.
Since ``2.3`` a :meth:`~elasticsearch.Elasticsearch.reindex` api is
available as part of elasticsearch itself. It is recommended to use the api
instead of this helper wherever possible. The helper is here mostly for
backwards compatibility and for situations where more flexibility is
needed.
.. note::
This helper doesn't transfer mappings, just the data.
:arg client: instance of :class:`~elasticsearch.Elasticsearch` to use (for
read if `target_client` is specified as well)
:arg source_index: index (or list of indices) to read documents from
:arg target_index: name of the index in the target cluster to populate
:arg query: body for the :meth:`~elasticsearch.Elasticsearch.search` api
:arg target_client: optional, is specified will be used for writing (thus
enabling reindex between clusters)
:arg chunk_size: number of docs in one chunk sent to es (default: 500)
:arg scroll: Specify how long a consistent view of the index should be
maintained for scrolled search
:arg scan_kwargs: additional kwargs to be passed to
:func:`~elasticsearch.helpers.scan`
:arg bulk_kwargs: additional kwargs to be passed to
:func:`~elasticsearch.helpers.bulk`
"""
target_client = client if target_client is None else target_client
docs = scan(client, query=query, index=source_index, scroll=scroll, **scan_kwargs)
def _change_doc_index(hits, index):
for h in hits:
h["_index"] = index
if "fields" in h:
h.update(h.pop("fields"))
yield h
kwargs = {"stats_only": True}
kwargs.update(bulk_kwargs)
return bulk(
target_client,
_change_doc_index(docs, target_index),
chunk_size=chunk_size,
**kwargs
)
def _add_helper_meta_to_kwargs(kwargs, helper_meta):
params = (kwargs or {}).pop("params", {})
params["__elastic_client_meta"] = (("h", helper_meta),)
kwargs["params"] = params
return kwargs

31
infrastructure/sandbox/Data/lambda/elasticsearch/helpers/errors.py
View file

@ -1,31 +0,0 @@
# Licensed to Elasticsearch B.V. under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch B.V. licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
from ..exceptions import ElasticsearchException
class BulkIndexError(ElasticsearchException):
@property
def errors(self):
""" List of errors from execution of the last chunk. """
return self.args[1]
class ScanError(ElasticsearchException):
def __init__(self, scroll_id, *args, **kwargs):
super(ScanError, self).__init__(*args, **kwargs)
self.scroll_id = scroll_id

81
infrastructure/sandbox/Data/lambda/elasticsearch/helpers/test.py
View file

@ -1,81 +0,0 @@
# Licensed to Elasticsearch B.V. under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch B.V. licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
import time
import os
try:
# python 2.6
from unittest2 import TestCase, SkipTest
except ImportError:
from unittest import TestCase, SkipTest
from elasticsearch import Elasticsearch
from elasticsearch.exceptions import ConnectionError
def get_test_client(nowait=False, **kwargs):
# construct kwargs from the environment
kw = {"timeout": 5}
if "TEST_ES_CONNECTION" in os.environ:
from elasticsearch import connection
kw["connection_class"] = getattr(connection, os.environ["TEST_ES_CONNECTION"])
kw.update(kwargs)
client = Elasticsearch([os.environ.get("TEST_ES_SERVER", {})], **kw)
# wait for yellow status
for _ in range(1 if nowait else 1):
try:
client.cluster.health(wait_for_status="yellow")
return client
except ConnectionError:
time.sleep(0.1)
else:
# timeout
raise SkipTest("Elasticsearch failed to start.")
def _get_version(version_string):
if "." not in version_string:
return ()
version = version_string.strip().split(".")
return tuple(int(v) if v.isdigit() else 999 for v in version)
class ElasticsearchTestCase(TestCase):
@staticmethod
def _get_client():
return get_test_client()
@classmethod
def setUpClass(cls):
super(ElasticsearchTestCase, cls).setUpClass()
cls.client = cls._get_client()
def tearDown(self):
super(ElasticsearchTestCase, self).tearDown()
self.client.indices.delete(index="*", ignore=404)
self.client.indices.delete_template(name="*", ignore=404)
@property
def es_version(self):
if not hasattr(self, "_es_version"):
version_string = self.client.info()["version"]["number"]
self._es_version = _get_version(version_string)
return self._es_version

156
infrastructure/sandbox/Data/lambda/elasticsearch/serializer.py
View file

@ -1,156 +0,0 @@
# Licensed to Elasticsearch B.V. under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch B.V. licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
try:
import simplejson as json
except ImportError:
import json
import uuid
from datetime import date, datetime
from decimal import Decimal
from .exceptions import SerializationError, ImproperlyConfigured
from .compat import string_types
INTEGER_TYPES = ()
FLOAT_TYPES = (Decimal,)
TIME_TYPES = (date, datetime)
try:
import numpy as np
INTEGER_TYPES += (
np.int_,
np.intc,
np.int8,
np.int16,
np.int32,
np.int64,
np.uint8,
np.uint16,
np.uint32,
np.uint64,
)
FLOAT_TYPES += (
np.float_,
np.float16,
np.float32,
np.float64,
)
except ImportError:
np = None
try:
import pandas as pd
TIME_TYPES += (pd.Timestamp,)
except ImportError:
pd = None
class TextSerializer(object):
mimetype = "text/plain"
def loads(self, s):
return s
def dumps(self, data):
if isinstance(data, string_types):
return data
raise SerializationError("Cannot serialize %r into text." % data)
class JSONSerializer(object):
mimetype = "application/json"
def default(self, data):
if isinstance(data, TIME_TYPES):
return data.isoformat()
elif isinstance(data, uuid.UUID):
return str(data)
elif isinstance(data, FLOAT_TYPES):
return float(data)
elif INTEGER_TYPES and isinstance(data, INTEGER_TYPES):
return int(data)
# Special cases for numpy and pandas types
elif np:
if isinstance(data, np.bool_):
return bool(data)
elif isinstance(data, np.datetime64):
return data.item().isoformat()
elif isinstance(data, np.ndarray):
return data.tolist()
if pd:
if isinstance(data, (pd.Series, pd.Categorical)):
return data.tolist()
elif hasattr(pd, "NA") and pd.isna(data):
return None
raise TypeError("Unable to serialize %r (type: %s)" % (data, type(data)))
def loads(self, s):
try:
return json.loads(s)
except (ValueError, TypeError) as e:
raise SerializationError(s, e)
def dumps(self, data):
# don't serialize strings
if isinstance(data, string_types):
return data
try:
return json.dumps(
data, default=self.default, ensure_ascii=False, separators=(",", ":")
)
except (ValueError, TypeError) as e:
raise SerializationError(data, e)
DEFAULT_SERIALIZERS = {
JSONSerializer.mimetype: JSONSerializer(),
TextSerializer.mimetype: TextSerializer(),
}
class Deserializer(object):
def __init__(self, serializers, default_mimetype="application/json"):
try:
self.default = serializers[default_mimetype]
except KeyError:
raise ImproperlyConfigured(
"Cannot find default serializer (%s)" % default_mimetype
)
self.serializers = serializers
def loads(self, s, mimetype=None):
if not mimetype:
deserializer = self.default
else:
# split out charset
mimetype, _, _ = mimetype.partition(";")
try:
deserializer = self.serializers[mimetype]
except KeyError:
raise SerializationError(
"Unknown mimetype, unable to deserialize: %s" % mimetype
)
return deserializer.loads(s)

450
infrastructure/sandbox/Data/lambda/elasticsearch/transport.py
View file

@ -1,450 +0,0 @@
# Licensed to Elasticsearch B.V. under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch B.V. licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
import time
from platform import python_version
from itertools import chain
from .connection import Urllib3HttpConnection
from .connection_pool import ConnectionPool, DummyConnectionPool
from .serializer import JSONSerializer, Deserializer, DEFAULT_SERIALIZERS
from .exceptions import (
ConnectionError,
TransportError,
SerializationError,
ConnectionTimeout,
)
from .utils import _client_meta_version
def get_host_info(node_info, host):
"""
Simple callback that takes the node info from `/_cluster/nodes` and a
parsed connection information and return the connection information. If
`None` is returned this node will be skipped.
Useful for filtering nodes (by proximity for example) or if additional
information needs to be provided for the :class:`~elasticsearch.Connection`
class. By default master only nodes are filtered out since they shouldn't
typically be used for API operations.
:arg node_info: node information from `/_cluster/nodes`
:arg host: connection information (host, port) extracted from the node info
"""
# ignore master only nodes
if node_info.get("roles", []) == ["master"]:
return None
return host
class Transport(object):
"""
Encapsulation of transport-related to logic. Handles instantiation of the
individual connections as well as creating a connection pool to hold them.
Main interface is the `perform_request` method.
"""
def __init__(
self,
hosts,
connection_class=Urllib3HttpConnection,
connection_pool_class=ConnectionPool,
host_info_callback=get_host_info,
sniff_on_start=False,
sniffer_timeout=None,
sniff_timeout=0.1,
sniff_on_connection_fail=False,
serializer=JSONSerializer(),
serializers=None,
default_mimetype="application/json",
max_retries=3,
retry_on_status=(502, 503, 504),
retry_on_timeout=False,
send_get_body_as="GET",
meta_header=True,
**kwargs
):
"""
:arg hosts: list of dictionaries, each containing keyword arguments to
create a `connection_class` instance
:arg connection_class: subclass of :class:`~elasticsearch.Connection` to use
:arg connection_pool_class: subclass of :class:`~elasticsearch.ConnectionPool` to use
:arg host_info_callback: callback responsible for taking the node information from
`/_cluser/nodes`, along with already extracted information, and
producing a list of arguments (same as `hosts` parameter)
:arg sniff_on_start: flag indicating whether to obtain a list of nodes
from the cluser at startup time
:arg sniffer_timeout: number of seconds between automatic sniffs
:arg sniff_on_connection_fail: flag controlling if connection failure triggers a sniff
:arg sniff_timeout: timeout used for the sniff request - it should be a
fast api call and we are talking potentially to more nodes so we want
to fail quickly. Not used during initial sniffing (if
``sniff_on_start`` is on) when the connection still isn't
initialized.
:arg serializer: serializer instance
:arg serializers: optional dict of serializer instances that will be
used for deserializing data coming from the server. (key is the mimetype)
:arg default_mimetype: when no mimetype is specified by the server
response assume this mimetype, defaults to `'application/json'`
:arg max_retries: maximum number of retries before an exception is propagated
:arg retry_on_status: set of HTTP status codes on which we should retry
on a different node. defaults to ``(502, 503, 504)``
:arg retry_on_timeout: should timeout trigger a retry on different
node? (default `False`)
:arg send_get_body_as: for GET requests with body this option allows
you to specify an alternate way of execution for environments that
don't support passing bodies with GET requests. If you set this to
'POST' a POST method will be used instead, if to 'source' then the body
will be serialized and passed as a query parameter `source`.
:arg meta_header: If True will send the 'X-Elastic-Client-Meta' HTTP header containing
simple client metadata. Setting to False will disable the header. Defaults to True.
Any extra keyword arguments will be passed to the `connection_class`
when creating and instance unless overridden by that connection's
options provided as part of the hosts parameter.
"""
if not isinstance(meta_header, bool):
raise TypeError("meta_header must be of type bool")
# serialization config
_serializers = DEFAULT_SERIALIZERS.copy()
# if a serializer has been specified, use it for deserialization as well
_serializers[serializer.mimetype] = serializer
# if custom serializers map has been supplied, override the defaults with it
if serializers:
_serializers.update(serializers)
# create a deserializer with our config
self.deserializer = Deserializer(_serializers, default_mimetype)
self.max_retries = max_retries
self.retry_on_timeout = retry_on_timeout
self.retry_on_status = retry_on_status
self.send_get_body_as = send_get_body_as
self.meta_header = meta_header
# data serializer
self.serializer = serializer
# store all strategies...
self.connection_pool_class = connection_pool_class
self.connection_class = connection_class
# ...save kwargs to be passed to the connections
self.kwargs = kwargs
self.hosts = hosts
# ...and instantiate them
self.set_connections(hosts)
# retain the original connection instances for sniffing
self.seed_connections = self.connection_pool.connections[:]
# Don't enable sniffing on Cloud instances.
if kwargs.get("cloud_id", False):
sniff_on_start = False
sniff_on_connection_fail = False
# sniffing data
self.sniffer_timeout = sniffer_timeout
self.sniff_on_connection_fail = sniff_on_connection_fail
self.last_sniff = time.time()
self.sniff_timeout = sniff_timeout
# callback to construct host dict from data in /_cluster/nodes
self.host_info_callback = host_info_callback
if sniff_on_start:
self.sniff_hosts(True)
# Create the default metadata for the x-elastic-client-meta
# HTTP header. Only requires adding the (service, service_version)
# tuple to the beginning of the client_meta
from . import __versionstr__
self._client_meta = (
("es", _client_meta_version(__versionstr__)),
("py", _client_meta_version(python_version())),
("t", _client_meta_version(__versionstr__)),
)
# Grab the 'HTTP_CLIENT_META' property from the connection class
http_client_meta = getattr(connection_class, "HTTP_CLIENT_META", None)
if http_client_meta:
self._client_meta += (http_client_meta,)
def add_connection(self, host):
"""
Create a new :class:`~elasticsearch.Connection` instance and add it to the pool.
:arg host: kwargs that will be used to create the instance
"""
self.hosts.append(host)
self.set_connections(self.hosts)
def set_connections(self, hosts):
"""
Instantiate all the connections and create new connection pool to hold them.
Tries to identify unchanged hosts and re-use existing
:class:`~elasticsearch.Connection` instances.
:arg hosts: same as `__init__`
"""
# construct the connections
def _create_connection(host):
# if this is not the initial setup look at the existing connection
# options and identify connections that haven't changed and can be
# kept around.
if hasattr(self, "connection_pool"):
for (connection, old_host) in self.connection_pool.connection_opts:
if old_host == host:
return connection
# previously unseen params, create new connection
kwargs = self.kwargs.copy()
kwargs.update(host)
return self.connection_class(**kwargs)
connections = map(_create_connection, hosts)
connections = list(zip(connections, hosts))
if len(connections) == 1:
self.connection_pool = DummyConnectionPool(connections)
else:
# pass the hosts dicts to the connection pool to optionally extract parameters from
self.connection_pool = self.connection_pool_class(
connections, **self.kwargs
)
def get_connection(self):
"""
Retreive a :class:`~elasticsearch.Connection` instance from the
:class:`~elasticsearch.ConnectionPool` instance.
"""
if self.sniffer_timeout:
if time.time() >= self.last_sniff + self.sniffer_timeout:
self.sniff_hosts()
return self.connection_pool.get_connection()
def _get_sniff_data(self, initial=False):
"""
Perform the request to get sniffins information. Returns a list of
dictionaries (one per node) containing all the information from the
cluster.
It also sets the last_sniff attribute in case of a successful attempt.
In rare cases it might be possible to override this method in your
custom Transport class to serve data from alternative source like
configuration management.
"""
previous_sniff = self.last_sniff
try:
# reset last_sniff timestamp
self.last_sniff = time.time()
# go through all current connections as well as the
# seed_connections for good measure
for c in chain(self.connection_pool.connections, self.seed_connections):
try:
# use small timeout for the sniffing request, should be a fast api call
_, headers, node_info = c.perform_request(
"GET",
"/_nodes/_all/http",
timeout=self.sniff_timeout if not initial else None,
)
node_info = self.deserializer.loads(
node_info, headers.get("content-type")
)
break
except (ConnectionError, SerializationError):
pass
else:
raise TransportError("N/A", "Unable to sniff hosts.")
except Exception:
# keep the previous value on error
self.last_sniff = previous_sniff
raise
return list(node_info["nodes"].values())
def _get_host_info(self, host_info):
host = {}
address = host_info.get("http", {}).get("publish_address")
# malformed or no address given
if not address or ":" not in address:
return None
host["host"], host["port"] = address.rsplit(":", 1)
host["port"] = int(host["port"])
return self.host_info_callback(host_info, host)
def sniff_hosts(self, initial=False):
"""
Obtain a list of nodes from the cluster and create a new connection
pool using the information retrieved.
To extract the node connection parameters use the ``nodes_to_host_callback``.
:arg initial: flag indicating if this is during startup
(``sniff_on_start``), ignore the ``sniff_timeout`` if ``True``
"""
node_info = self._get_sniff_data(initial)
hosts = list(filter(None, (self._get_host_info(n) for n in node_info)))
# we weren't able to get any nodes or host_info_callback blocked all -
# raise error.
if not hosts:
raise TransportError(
"N/A", "Unable to sniff hosts - no viable hosts found."
)
self.set_connections(hosts)
def mark_dead(self, connection):
"""
Mark a connection as dead (failed) in the connection pool. If sniffing
on failure is enabled this will initiate the sniffing process.
:arg connection: instance of :class:`~elasticsearch.Connection` that failed
"""
# mark as dead even when sniffing to avoid hitting this host during the sniff process
self.connection_pool.mark_dead(connection)
if self.sniff_on_connection_fail:
self.sniff_hosts()
def perform_request(self, method, url, headers=None, params=None, body=None):
"""
Perform the actual request. Retrieve a connection from the connection
pool, pass all the information to it's perform_request method and
return the data.
If an exception was raised, mark the connection as failed and retry (up
to `max_retries` times).
If the operation was succesful and the connection used was previously
marked as dead, mark it as live, resetting it's failure count.
:arg method: HTTP method to use
:arg url: absolute url (without host) to target
:arg headers: dictionary of headers, will be handed over to the
underlying :class:`~elasticsearch.Connection` class
:arg params: dictionary of query parameters, will be handed over to the
underlying :class:`~elasticsearch.Connection` class for serialization
:arg body: body of the request, will be serializes using serializer and
passed to the connection
"""
if body is not None:
body = self.serializer.dumps(body)
# some clients or environments don't support sending GET with body
if method in ("HEAD", "GET") and self.send_get_body_as != "GET":
# send it as post instead
if self.send_get_body_as == "POST":
method = "POST"
# or as source parameter
elif self.send_get_body_as == "source":
if params is None:
params = {}
params["source"] = body
body = None
if body is not None:
try:
body = body.encode("utf-8", "surrogatepass")
except (UnicodeDecodeError, AttributeError):
# bytes/str - no need to re-encode
pass
ignore = ()
timeout = None
if params:
timeout = params.pop("request_timeout", None)
ignore = params.pop("ignore", ())
if isinstance(ignore, int):
ignore = (ignore,)
client_meta = params.pop("__elastic_client_meta", ())
else:
client_meta = ()
if self.meta_header:
headers = headers or {}
client_meta = self._client_meta + client_meta
headers["x-elastic-client-meta"] = ",".join(
"%s=%s" % (k, v) for k, v in client_meta
)
for attempt in range(self.max_retries + 1):
connection = self.get_connection()
try:
# add a delay before attempting the next retry
# 0, 1, 3, 7, etc...
delay = 2 ** attempt - 1
time.sleep(delay)
status, headers_response, data = connection.perform_request(
method,
url,
params,
body,
headers=headers,
ignore=ignore,
timeout=timeout,
)
except TransportError as e:
if method == "HEAD" and e.status_code == 404:
return False
retry = False
if isinstance(e, ConnectionTimeout):
retry = self.retry_on_timeout
elif isinstance(e, ConnectionError):
retry = True
elif e.status_code in self.retry_on_status:
retry = True
if retry:
# only mark as dead if we are retrying
self.mark_dead(connection)
# raise exception on last retry
if attempt == self.max_retries:
raise
else:
raise
else:
# connection didn't fail, confirm it's live status
self.connection_pool.mark_live(connection)
if method == "HEAD":
return 200 <= status < 300
if data:
data = self.deserializer.loads(
data, headers_response.get("content-type")
)
return data
def close(self):
"""
Explicitly closes connections
"""
self.connection_pool.close()

31
infrastructure/sandbox/Data/lambda/elasticsearch/utils.py
View file

@ -1,31 +0,0 @@
# Licensed to Elasticsearch B.V. under one or more contributor
# license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright
# ownership. Elasticsearch B.V. licenses this file to you under
# the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
import re
def _client_meta_version(version):
"""Transforms a Python package version to one
compatible with 'X-Elastic-Client-Meta'. Essentially
replaces any pre-release information with a 'p' suffix.
"""
version, version_pre = re.match(
r"^([0-9][0-9.]*[0-9]|[0-9])(.*)$", version
).groups()
if version_pre:
version += "p"
return version

465
infrastructure/sandbox/Data/lambda/geohash.py
View file

@ -1,465 +0,0 @@
# coding: UTF-8
"""
Copyright (C) 2009 Hiroaki Kawai <kawai@iij.ad.jp>
"""
try:
import _geohash
except ImportError:
_geohash = None
__version__ = "0.8.5"
__all__ = ['encode','decode','decode_exactly','bbox', 'neighbors', 'expand']
_base32 = '0123456789bcdefghjkmnpqrstuvwxyz'
_base32_map = {}
for i in range(len(_base32)):
_base32_map[_base32[i]] = i
del i
LONG_ZERO = 0
import sys
if sys.version_info[0] < 3:
LONG_ZERO = long(0)
def _float_hex_to_int(f):
if f<-1.0 or f>=1.0:
return None
if f==0.0:
return 1,1
h = f.hex()
x = h.find("0x1.")
assert(x>=0)
p = h.find("p")
assert(p>0)
half_len = len(h[x+4:p])*4-int(h[p+1:])
if x==0:
r = (1<<half_len) + ((1<<(len(h[x+4:p])*4)) + int(h[x+4:p],16))
else:
r = (1<<half_len) - ((1<<(len(h[x+4:p])*4)) + int(h[x+4:p],16))
return r, half_len+1
def _int_to_float_hex(i, l):
if l==0:
return -1.0
half = 1<<(l-1)
s = int((l+3)/4)
if i >= half:
i = i-half
return float.fromhex(("0x0.%0"+str(s)+"xp1") % (i<<(s*4-l),))
else:
i = half-i
return float.fromhex(("-0x0.%0"+str(s)+"xp1") % (i<<(s*4-l),))
def _encode_i2c(lat,lon,lat_length,lon_length):
precision = int((lat_length+lon_length)/5)
if lat_length < lon_length:
a = lon
b = lat
else:
a = lat
b = lon
boost = (0,1,4,5,16,17,20,21)
ret = ''
for i in range(precision):
ret+=_base32[(boost[a&7]+(boost[b&3]<<1))&0x1F]
t = a>>3
a = b>>2
b = t
return ret[::-1]
def encode(latitude, longitude, precision=12):
if latitude >= 90.0 or latitude < -90.0:
raise Exception("invalid latitude.")
while longitude < -180.0:
longitude += 360.0
while longitude >= 180.0:
longitude -= 360.0
if _geohash:
basecode=_geohash.encode(latitude,longitude)
if len(basecode)>precision:
return basecode[0:precision]
return basecode+'0'*(precision-len(basecode))
xprecision=precision+1
lat_length = lon_length = int(xprecision*5/2)
if xprecision%2==1:
lon_length+=1
if hasattr(float, "fromhex"):
a = _float_hex_to_int(latitude/90.0)
o = _float_hex_to_int(longitude/180.0)
if a[1] > lat_length:
ai = a[0]>>(a[1]-lat_length)
else:
ai = a[0]<<(lat_length-a[1])
if o[1] > lon_length:
oi = o[0]>>(o[1]-lon_length)
else:
oi = o[0]<<(lon_length-o[1])
return _encode_i2c(ai, oi, lat_length, lon_length)[:precision]
lat = latitude/180.0
lon = longitude/360.0
if lat>0:
lat = int((1<<lat_length)*lat)+(1<<(lat_length-1))
else:
lat = (1<<lat_length-1)-int((1<<lat_length)*(-lat))
if lon>0:
lon = int((1<<lon_length)*lon)+(1<<(lon_length-1))
else:
lon = (1<<lon_length-1)-int((1<<lon_length)*(-lon))
return _encode_i2c(lat,lon,lat_length,lon_length)[:precision]
def _decode_c2i(hashcode):
lon = 0
lat = 0
bit_length = 0
lat_length = 0
lon_length = 0
for i in hashcode:
t = _base32_map[i]
if bit_length%2==0:
lon = lon<<3
lat = lat<<2
lon += (t>>2)&4
lat += (t>>2)&2
lon += (t>>1)&2
lat += (t>>1)&1
lon += t&1
lon_length+=3
lat_length+=2
else:
lon = lon<<2
lat = lat<<3
lat += (t>>2)&4
lon += (t>>2)&2
lat += (t>>1)&2
lon += (t>>1)&1
lat += t&1
lon_length+=2
lat_length+=3
bit_length+=5
return (lat,lon,lat_length,lon_length)
def decode(hashcode, delta=False):
'''
decode a hashcode and get center coordinate, and distance between center and outer border
'''
if _geohash:
(lat,lon,lat_bits,lon_bits) = _geohash.decode(hashcode)
latitude_delta = 90.0/(1<<lat_bits)
longitude_delta = 180.0/(1<<lon_bits)
latitude = lat + latitude_delta
longitude = lon + longitude_delta
if delta:
return latitude,longitude,latitude_delta,longitude_delta
return latitude,longitude
(lat,lon,lat_length,lon_length) = _decode_c2i(hashcode)
if hasattr(float, "fromhex"):
latitude_delta = 90.0/(1<<lat_length)
longitude_delta = 180.0/(1<<lon_length)
latitude = _int_to_float_hex(lat, lat_length) * 90.0 + latitude_delta
longitude = _int_to_float_hex(lon, lon_length) * 180.0 + longitude_delta
if delta:
return latitude,longitude,latitude_delta,longitude_delta
return latitude,longitude
lat = (lat<<1) + 1
lon = (lon<<1) + 1
lat_length += 1
lon_length += 1
latitude = 180.0*(lat-(1<<(lat_length-1)))/(1<<lat_length)
longitude = 360.0*(lon-(1<<(lon_length-1)))/(1<<lon_length)
if delta:
latitude_delta = 180.0/(1<<lat_length)
longitude_delta = 360.0/(1<<lon_length)
return latitude,longitude,latitude_delta,longitude_delta
return latitude,longitude
def decode_exactly(hashcode):
return decode(hashcode, True)
## hashcode operations below
def bbox(hashcode):
'''
decode a hashcode and get north, south, east and west border.
'''
if _geohash:
(lat,lon,lat_bits,lon_bits) = _geohash.decode(hashcode)
latitude_delta = 180.0/(1<<lat_bits)
longitude_delta = 360.0/(1<<lon_bits)
return {'s':lat,'w':lon,'n':lat+latitude_delta,'e':lon+longitude_delta}
(lat,lon,lat_length,lon_length) = _decode_c2i(hashcode)
if hasattr(float, "fromhex"):
latitude_delta = 180.0/(1<<lat_length)
longitude_delta = 360.0/(1<<lon_length)
latitude = _int_to_float_hex(lat, lat_length) * 90.0
longitude = _int_to_float_hex(lon, lon_length) * 180.0
return {"s":latitude, "w":longitude, "n":latitude+latitude_delta, "e":longitude+longitude_delta}
ret={}
if lat_length:
ret['n'] = 180.0*(lat+1-(1<<(lat_length-1)))/(1<<lat_length)
ret['s'] = 180.0*(lat-(1<<(lat_length-1)))/(1<<lat_length)
else: # can't calculate the half with bit shifts (negative shift)
ret['n'] = 90.0
ret['s'] = -90.0
if lon_length:
ret['e'] = 360.0*(lon+1-(1<<(lon_length-1)))/(1<<lon_length)
ret['w'] = 360.0*(lon-(1<<(lon_length-1)))/(1<<lon_length)
else: # can't calculate the half with bit shifts (negative shift)
ret['e'] = 180.0
ret['w'] = -180.0
return ret
def neighbors(hashcode):
if _geohash and len(hashcode)<25:
return _geohash.neighbors(hashcode)
(lat,lon,lat_length,lon_length) = _decode_c2i(hashcode)
ret = []
tlat = lat
for tlon in (lon-1, lon+1):
code = _encode_i2c(tlat,tlon,lat_length,lon_length)
if code:
ret.append(code)
tlat = lat+1
if not tlat >> lat_length:
for tlon in (lon-1, lon, lon+1):
ret.append(_encode_i2c(tlat,tlon,lat_length,lon_length))
tlat = lat-1
if tlat >= 0:
for tlon in (lon-1, lon, lon+1):
ret.append(_encode_i2c(tlat,tlon,lat_length,lon_length))
return ret
def expand(hashcode):
ret = neighbors(hashcode)
ret.append(hashcode)
return ret
def _uint64_interleave(lat32, lon32):
intr = 0
boost = (0,1,4,5,16,17,20,21,64,65,68,69,80,81,84,85)
for i in range(8):
intr = (intr<<8) + (boost[(lon32>>(28-i*4))%16]<<1) + boost[(lat32>>(28-i*4))%16]
return intr
def _uint64_deinterleave(ui64):
lat = lon = 0
boost = ((0,0),(0,1),(1,0),(1,1),(0,2),(0,3),(1,2),(1,3),
(2,0),(2,1),(3,0),(3,1),(2,2),(2,3),(3,2),(3,3))
for i in range(16):
p = boost[(ui64>>(60-i*4))%16]
lon = (lon<<2) + p[0]
lat = (lat<<2) + p[1]
return (lat, lon)
def encode_uint64(latitude, longitude):
if latitude >= 90.0 or latitude < -90.0:
raise ValueError("Latitude must be in the range of (-90.0, 90.0)")
while longitude < -180.0:
longitude += 360.0
while longitude >= 180.0:
longitude -= 360.0
if _geohash:
ui128 = _geohash.encode_int(latitude,longitude)
if _geohash.intunit == 64:
return ui128[0]
elif _geohash.intunit == 32:
return (ui128[0]<<32) + ui128[1]
elif _geohash.intunit == 16:
return (ui128[0]<<48) + (ui128[1]<<32) + (ui128[2]<<16) + ui128[3]
lat = int(((latitude + 90.0)/180.0)*(1<<32))
lon = int(((longitude+180.0)/360.0)*(1<<32))
return _uint64_interleave(lat, lon)
def decode_uint64(ui64):
if _geohash:
latlon = _geohash.decode_int(ui64 % 0xFFFFFFFFFFFFFFFF, LONG_ZERO)
if latlon:
return latlon
lat,lon = _uint64_deinterleave(ui64)
return (180.0*lat/(1<<32) - 90.0, 360.0*lon/(1<<32) - 180.0)
def expand_uint64(ui64, precision=50):
ui64 = ui64 & (0xFFFFFFFFFFFFFFFF << (64-precision))
lat,lon = _uint64_deinterleave(ui64)
lat_grid = 1<<(32-int(precision/2))
lon_grid = lat_grid>>(precision%2)
if precision<=2: # expand becomes to the whole range
return []
ranges = []
if lat & lat_grid:
if lon & lon_grid:
ui64 = _uint64_interleave(lat-lat_grid, lon-lon_grid)
ranges.append((ui64, ui64 + (1<<(64-precision+2))))
if precision%2==0:
# lat,lon = (1, 1) and even precision
ui64 = _uint64_interleave(lat-lat_grid, lon+lon_grid)
ranges.append((ui64, ui64 + (1<<(64-precision+1))))
if lat + lat_grid < 0xFFFFFFFF:
ui64 = _uint64_interleave(lat+lat_grid, lon-lon_grid)
ranges.append((ui64, ui64 + (1<<(64-precision))))
ui64 = _uint64_interleave(lat+lat_grid, lon)
ranges.append((ui64, ui64 + (1<<(64-precision))))
ui64 = _uint64_interleave(lat+lat_grid, lon+lon_grid)
ranges.append((ui64, ui64 + (1<<(64-precision))))
else:
# lat,lon = (1, 1) and odd precision
if lat + lat_grid < 0xFFFFFFFF:
ui64 = _uint64_interleave(lat+lat_grid, lon-lon_grid)
ranges.append((ui64, ui64 + (1<<(64-precision+1))))
ui64 = _uint64_interleave(lat+lat_grid, lon+lon_grid)
ranges.append((ui64, ui64 + (1<<(64-precision))))
ui64 = _uint64_interleave(lat, lon+lon_grid)
ranges.append((ui64, ui64 + (1<<(64-precision))))
ui64 = _uint64_interleave(lat-lat_grid, lon+lon_grid)
ranges.append((ui64, ui64 + (1<<(64-precision))))
else:
ui64 = _uint64_interleave(lat-lat_grid, lon)
ranges.append((ui64, ui64 + (1<<(64-precision+2))))
if precision%2==0:
# lat,lon = (1, 0) and odd precision
ui64 = _uint64_interleave(lat-lat_grid, lon-lon_grid)
ranges.append((ui64, ui64 + (1<<(64-precision+1))))
if lat + lat_grid < 0xFFFFFFFF:
ui64 = _uint64_interleave(lat+lat_grid, lon-lon_grid)
ranges.append((ui64, ui64 + (1<<(64-precision))))
ui64 = _uint64_interleave(lat+lat_grid, lon)
ranges.append((ui64, ui64 + (1<<(64-precision))))
ui64 = _uint64_interleave(lat+lat_grid, lon+lon_grid)
ranges.append((ui64, ui64 + (1<<(64-precision))))
else:
# lat,lon = (1, 0) and odd precision
if lat + lat_grid < 0xFFFFFFFF:
ui64 = _uint64_interleave(lat+lat_grid, lon)
ranges.append((ui64, ui64 + (1<<(64-precision+1))))
ui64 = _uint64_interleave(lat+lat_grid, lon-lon_grid)
ranges.append((ui64, ui64 + (1<<(64-precision))))
ui64 = _uint64_interleave(lat, lon-lon_grid)
ranges.append((ui64, ui64 + (1<<(64-precision))))
ui64 = _uint64_interleave(lat-lat_grid, lon-lon_grid)
ranges.append((ui64, ui64 + (1<<(64-precision))))
else:
if lon & lon_grid:
ui64 = _uint64_interleave(lat, lon-lon_grid)
ranges.append((ui64, ui64 + (1<<(64-precision+2))))
if precision%2==0:
# lat,lon = (0, 1) and even precision
ui64 = _uint64_interleave(lat, lon+lon_grid)
ranges.append((ui64, ui64 + (1<<(64-precision+1))))
if lat > 0:
ui64 = _uint64_interleave(lat-lat_grid, lon-lon_grid)
ranges.append((ui64, ui64 + (1<<(64-precision))))
ui64 = _uint64_interleave(lat-lat_grid, lon)
ranges.append((ui64, ui64 + (1<<(64-precision))))
ui64 = _uint64_interleave(lat-lat_grid, lon+lon_grid)
ranges.append((ui64, ui64 + (1<<(64-precision))))
else:
# lat,lon = (0, 1) and odd precision
if lat > 0:
ui64 = _uint64_interleave(lat-lat_grid, lon-lon_grid)
ranges.append((ui64, ui64 + (1<<(64-precision+1))))
ui64 = _uint64_interleave(lat-lat_grid, lon+lon_grid)
ranges.append((ui64, ui64 + (1<<(64-precision))))
ui64 = _uint64_interleave(lat, lon+lon_grid)
ranges.append((ui64, ui64 + (1<<(64-precision))))
ui64 = _uint64_interleave(lat+lat_grid, lon+lon_grid)
ranges.append((ui64, ui64 + (1<<(64-precision))))
else:
ui64 = _uint64_interleave(lat, lon)
ranges.append((ui64, ui64 + (1<<(64-precision+2))))
if precision%2==0:
# lat,lon = (0, 0) and even precision
ui64 = _uint64_interleave(lat, lon-lon_grid)
ranges.append((ui64, ui64 + (1<<(64-precision+1))))
if lat > 0:
ui64 = _uint64_interleave(lat-lat_grid, lon-lon_grid)
ranges.append((ui64, ui64 + (1<<(64-precision))))
ui64 = _uint64_interleave(lat-lat_grid, lon)
ranges.append((ui64, ui64 + (1<<(64-precision))))
ui64 = _uint64_interleave(lat-lat_grid, lon+lon_grid)
ranges.append((ui64, ui64 + (1<<(64-precision))))
else:
# lat,lon = (0, 0) and odd precision
if lat > 0:
ui64 = _uint64_interleave(lat-lat_grid, lon)
ranges.append((ui64, ui64 + (1<<(64-precision+1))))
ui64 = _uint64_interleave(lat-lat_grid, lon-lon_grid)
ranges.append((ui64, ui64 + (1<<(64-precision))))
ui64 = _uint64_interleave(lat, lon-lon_grid)
ranges.append((ui64, ui64 + (1<<(64-precision))))
ui64 = _uint64_interleave(lat+lat_grid, lon-lon_grid)
ranges.append((ui64, ui64 + (1<<(64-precision))))
ranges.sort()
# merge the conditions
shrink = []
prev = None
for i in ranges:
if prev:
if prev[1] != i[0]:
shrink.append(prev)
prev = i
else:
prev = (prev[0], i[1])
else:
prev = i
shrink.append(prev)
ranges = []
for i in shrink:
a,b=i
if a == 0:
a = None # we can remove the condition because it is the lowest value
if b == 0x10000000000000000:
b = None # we can remove the condition because it is the highest value
ranges.append((a,b))
return ranges

509
infrastructure/sandbox/Data/lambda/geoip.py
View file

@ -1,509 +0,0 @@
import sys
import mmap
import socket
import urllib
from threading import Lock
from datetime import datetime
from struct import Struct
MMDB_METADATA_START = b'\xAB\xCD\xEFMaxMind.com'
MMDB_METADATA_BLOCK_MAX_SIZE = 131072
MMDB_DATA_SECTION_SEPARATOR = 16
_int_unpack = Struct('>I').unpack
_long_unpack = Struct('>Q').unpack
_short_unpack = Struct('>H').unpack
def _native_str(x):
"""Attempts to coerce a string into native if it's ASCII safe."""
try:
return str(x)
except UnicodeError:
return x
def pack_ip(ip):
"""Given an IP string, converts it into packed format for internal
usage.
"""
for fmly in socket.AF_INET, socket.AF_INET6:
try:
return socket.inet_pton(fmly, ip)
except socket.error:
continue
raise ValueError('Malformed IP address')
class DatabaseInfo(object):
"""Provides information about the GeoIP database."""
def __init__(self, filename=None, date=None,
internal_name=None, provider=None):
#: If available the filename which backs the database.
self.filename = filename
#: Optionally the build date of the database as datetime object.
self.date = date
#: Optionally the internal name of the database.
self.internal_name = internal_name
#: Optionally the name of the database provider.
self.provider = provider
def __repr__(self):
return '<%s filename=%r date=%r internal_name=%r provider=%r>' % (
self.__class__.__name__,
self.filename,
self.date,
self.internal_name,
self.provider,
)
class IPInfo(object):
"""Provides information about the located IP as returned by
:meth:`Database.lookup`.
"""
__slots__ = ('ip', '_data')
def __init__(self, ip, data):
#: The IP that was looked up.
self.ip = ip
self._data = data
@property
def country(self):
"""The country code as ISO code if available."""
if 'country' in self._data:
return _native_str(self._data['country']['iso_code'])
@property
def continent(self):
"""The continent as ISO code if available."""
if 'continent' in self._data:
return _native_str(self._data['continent']['code'])
@property
def subdivisions(self):
"""The subdivisions as a list of ISO codes as an immutable set."""
return frozenset(_native_str(x['iso_code']) for x in
self._data.get('subdivisions') or () if 'iso_code'
in x)
@property
def timezone(self):
"""The timezone if available as tzinfo name."""
if 'location' in self._data:
return _native_str(self._data['location'].get('time_zone'))
@property
def location(self):
"""The location as ``(lat, long)`` tuple if available."""
if 'location' in self._data:
lat = self._data['location'].get('latitude')
long = self._data['location'].get('longitude')
if lat is not None and long is not None:
return lat, long
def to_dict(self):
"""A dict representation of the available information. This
is a dictionary with the same keys as the attributes of this
object.
"""
return {
'ip': self.ip,
'country': self.country,
'continent': self.continent,
'subdivisions': self.subdivisions,
'timezone': self.timezone,
'location': self.location,
}
def get_info_dict(self):
"""Returns the internal info dictionary. For a maxmind database
this is the metadata dictionary.
"""
return self._data
def __hash__(self):
return hash(self.addr)
def __eq__(self, other):
return type(self) is type(other) and self.addr == other.addr
def __ne__(self, other):
return not self.__eq__(other)
def __repr__(self):
return ('<IPInfo ip=%r country=%r continent=%r '
'subdivisions=%r timezone=%r location=%r>') % (
self.ip,
self.country,
self.continent,
self.subdivisions,
self.timezone,
self.location,
)
class Database(object):
"""Provides access to a GeoIP database. This is an abstract class
that is implemented by different providers. The :func:`open_database`
function can be used to open a MaxMind database.
Example usage::
from geoip import open_database
with open_database('data/GeoLite2-City.mmdb') as db:
match = db.lookup_mine()
print 'My IP info:', match
"""
def __init__(self):
self.closed = False
def __enter__(self):
return self
def __exit__(self, exc_type, exc_value, tb):
self.close()
def close(self):
"""Closes the database. The whole object can also be used as a
context manager. Databases that are packaged up (such as the
:data:`geolite2` database) do not need to be closed.
"""
self.closed = True
def get_info(self):
"""Returns an info object about the database. This can be used to
check for the build date of the database or what provides the GeoIP
data.
:rtype: :class:`DatabaseInfo`
"""
raise NotImplementedError('This database does not provide info')
def get_metadata(self):
"""Return the metadata dictionary of the loaded database. This
dictionary is specific to the database provider.
"""
raise NotImplementedError('This database does not provide metadata')
def lookup(self, ip_addr):
"""Looks up the IP information in the database and returns a
:class:`IPInfo`. If it does not exist, `None` is returned. What
IP addresses are supported is specific to the GeoIP provider.
:rtype: :class:`IPInfo`
"""
if self.closed:
raise RuntimeError('Database is closed.')
return self._lookup(ip_addr)
def lookup_mine(self):
"""Looks up the computer's IP by asking a web service and then
checks the database for a match.
:rtype: :class:`IPInfo`
"""
ip = urllib.urlopen('http://icanhazip.com/').read().strip()
return self.lookup(ip)
class MaxMindDatabase(Database):
"""Provides access to a maxmind database."""
def __init__(self, filename, buf, md):
Database.__init__(self)
self.filename = filename
self.is_ipv6 = md['ip_version'] == 6
self.nodes = md['node_count']
self.record_size = md['record_size']
self.node_size = int(self.record_size / 4)
self.db_size = self.nodes * self.node_size
self._buf = buf
self._md = md
self._reader = _MaxMindParser(buf, self.db_size)
self._ipv4_start = None
def close(self):
Database.close(self)
self._buf.close()
def get_metadata(self):
return self._md
def get_info(self):
return DatabaseInfo(
filename=self.filename,
date=datetime.utcfromtimestamp(self._md['build_epoch']),
internal_name=_native_str(self._md['database_type']),
provider='maxmind',
)
def _lookup(self, ip_addr):
packed_addr = pack_ip(ip_addr)
bits = len(packed_addr) * 8
node = self._find_start_node(bits)
seen = set()
for i in range(bits):
if node >= self.nodes:
break
bit = (packed_addr[i >> 3] >> (7 - (i % 8))) & 1
node = self._parse_node(node, bit)
if node in seen:
raise LookupError('Circle in tree detected')
seen.add(node)
if node > self.nodes:
offset = node - self.nodes + self.db_size
return IPInfo(ip_addr, self._reader.read(offset)[0])
def _find_start_node(self, bits):
if bits == 128 or not self.is_ipv6:
return 0
if self._ipv4_start is not None:
return self._ipv4_start
# XXX: technically the next code is racy if used concurrently but
# the worst thing that can happen is that the ipv4 start node is
# calculated multiple times.
node = 0
for netmask in range(96):
if node >= self.nodes:
break
node = self._parse_node(netmask, 0)
self._ipv4_start = node
return node
def _parse_node(self, node, index):
offset = node * self.node_size
if self.record_size == 24:
offset += index * 3
bytes = b'\x00' + self._buf[offset:offset + 3]
elif self.record_size == 28:
b = ord(self._buf[offset + 3:offset + 4])
if index:
b &= 0x0F
else:
b = (0xF0 & b) >> 4
offset += index * 4
bytes = chr(b).encode('utf8') + self._buf[offset:offset + 3]
elif self.record_size == 32:
offset += index * 4
bytes = self._buf[offset:offset + 4]
else:
raise LookupError('Invalid record size')
return _int_unpack(bytes)[0]
def __repr__(self):
return '<%s %r>' % (
self.__class__.__name__,
self.filename,
)
class PackagedDatabase(Database):
"""Provides access to a packaged database. Upon first usage the
system will import the provided package and invoke the ``loader``
function to construct the actual database object.
This is used for instance to implement the ``geolite2`` database
that is provided.
"""
def __init__(self, name, package, pypi_name=None):
Database.__init__(self)
self.name = name
self.package = package
self.pypi_name = pypi_name
self._lock = Lock()
self._db = None
def _load_database(self):
try:
mod = __import__(self.package, None, None, ['loader'])
except ImportError:
msg = 'Cannot use packaged database "%s" ' \
'because package "%s" is not available.' % (self.name,
self.package)
if self.pypi_name is not None:
msg += ' It\'s provided by PyPI package "%s"' % self.pypi_name
raise RuntimeError(msg)
return mod.loader(self, sys.modules[__name__])
def _get_actual_db(self):
if self._db is not None:
return self._db
with self._lock:
if self._db is not None:
return self._db
rv = self._load_database()
self._db = rv
return rv
def close(self):
pass
def get_info(self):
return self._get_actual_db().get_info()
def get_metadata(self):
return self._get_actual_db().get_metadata()
def lookup(self, ip_addr):
return self._get_actual_db().lookup(ip_addr)
def __repr__(self):
return '<%s %r>' % (
self.__class__.__name__,
self.name,
)
#: Provides access to the geolite2 cities database. In order to use this
#: database the ``python-geoip-geolite2`` package needs to be installed.
geolite2 = PackagedDatabase('geolite2', '_geoip_geolite2',
pypi_name='python-geoip-geolite2')
def _read_mmdb_metadata(buf):
"""Reads metadata from a given memory mapped buffer."""
offset = buf.rfind(MMDB_METADATA_START,
buf.size() - MMDB_METADATA_BLOCK_MAX_SIZE)
if offset < 0:
raise ValueError('Could not find metadata')
offset += len(MMDB_METADATA_START)
return _MaxMindParser(buf, offset).read(offset)[0]
def make_struct_parser(code):
struct = Struct('>' + code)
def unpack_func(self, size, offset):
new_offset = offset + struct.size
bytes = self._buf[offset:new_offset].rjust(struct.size, b'\x00')
value = struct.unpack(bytes)[0]
return value, new_offset
return unpack_func
class _MaxMindParser(object):
def __init__(self, buf, data_offset=0):
self._buf = buf
self._data_offset = data_offset
def _parse_ptr(self, size, offset):
ptr_size = ((size >> 3) & 0x3) + 1
bytes = self._buf[offset:offset + ptr_size]
if ptr_size != 4:
bytes = chr(size & 0x7).encode('utf8') + bytes
ptr = (
_int_unpack(bytes.rjust(4, b'\x00'))[0] +
self._data_offset +
MMDB_DATA_SECTION_SEPARATOR +
(0, 2048, 526336, 0)[ptr_size - 1]
)
return self.read(ptr)[0], offset + ptr_size
def _parse_str(self, size, offset):
bytes = self._buf[offset:offset + size]
return bytes.decode('utf-8', 'replace'), offset + size
_parse_double = make_struct_parser('d')
def _parse_bytes(self, size, offset):
return self._buf[offset:offset + size], offset + size
def _parse_uint(self, size, offset):
bytes = self._buf[offset:offset + size]
return _long_unpack(bytes.rjust(8, b'\x00'))[0], offset + size
def _parse_dict(self, size, offset):
container = {}
for _ in range(size):
key, offset = self.read(offset)
value, offset = self.read(offset)
container[key] = value
return container, offset
_parse_int32 = make_struct_parser('i')
def _parse_list(self, size, offset):
rv = [None] * size
for idx in range(size):
rv[idx], offset = self.read(offset)
return rv, offset
def _parse_error(self, size, offset):
raise AssertionError('Read invalid type code')
def _parse_bool(self, size, offset):
return size != 0, offset
_parse_float = make_struct_parser('f')
_callbacks = (
_parse_error, # 0 <extended>
_parse_ptr, # 1 pointer
_parse_str, # 2 utf-8 string
_parse_double, # 3 double
_parse_bytes, # 4 bytes
_parse_uint, # 5 uint16
_parse_uint, # 6 uint32
_parse_dict, # 7 map
_parse_int32, # 8 int32
_parse_uint, # 9 uint64
_parse_uint, # 10 uint128
_parse_list, # 11 array
_parse_error, # 12 <container>
_parse_error, # 13 <end_marker>
_parse_bool, # 14 boolean
_parse_float, # 15 float
)
def read(self, offset):
new_offset = offset + 1
byte = ord(self._buf[offset:new_offset])
size = byte & 0x1f
ty = byte >> 5
if ty == 0:
byte = ord(self._buf[new_offset:new_offset + 1])
ty = byte + 7
new_offset += 1
if ty != 1 and size >= 29:
to_read = size - 28
bytes = self._buf[new_offset:new_offset + to_read]
new_offset += to_read
if size == 29:
size = 29 + ord(bytes)
elif size == 30:
size = 285 + _short_unpack(bytes)[0]
elif size > 30:
size = 65821 + _int_unpack(bytes.rjust(4, b'\x00'))[0]
return self._callbacks[ty](self, size, new_offset)
def open_database(filename):
"""Open a given database. This currently only supports maxmind
databases (mmdb). If the file cannot be opened an ``IOError`` is
raised.
"""
with open(filename, 'rb') as f:
buf = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ)
md = _read_mmdb_metadata(buf)
return MaxMindDatabase(filename, buf, md)

161
infrastructure/sandbox/Data/lambda/jpgrid.py
View file

@ -1,161 +0,0 @@
# coding: UTF-8
# Coder for Japanese grid square code. (JIS C 6304 / JIS X 0410)
# 行政管理庁告示第143号 http://www.stat.go.jp/data/mesh/
def _encode_i2c(lat, lon, base1):
t=[]
while base1>80:
t.append(1 + (lat&1)*2 + (lon&1))
lat = lat>>1
lon = lon>>1
base1 = base1>>1
if base1==80:
t.append(lon%10)
t.append(lat%10)
lat = int(lat/10)
lon = int(lon/10)
base1 = int(base1/10)
elif base1==16: # Uni5
t.append(1 + (lat&1)*2 + (lon&1))
lat = lat>>1
lon = lon>>1
base1 = base1>>1
elif base1==40: # Uni2
t.append(5)
t.append(lon%5*2)
t.append(lat%5*2)
lat = int(lat/5)
lon = int(lon/5)
base1 = int(base1/5)
if base1==8:
t.append(lon%8)
t.append(lat%8)
lat = lat>>3
lon = lon>>3
base1 = base1>>3
t.append(lon)
t.append(lat)
t.reverse()
return ''.join([str(i) for i in t])
def encode(latitude, longitude, base1=80):
return _encode_i2c(int(latitude*base1*1.5), int(longitude*base1-100.0*base1), base1)
#def _encode_i2c(lat, lon, base1):
def _decode_c2i(gridcode):
base1 = 1
lat = lon = 0
codelen = len(gridcode)
if codelen>0:
lat = int(gridcode[0:2])
lon = int(gridcode[2:4])
if codelen>4:
lat = (lat<<3) + int(gridcode[4:5])
lon = (lon<<3) + int(gridcode[5:6])
base1 = base1<<3
if codelen>6:
if codelen==7:
i = int(gridcode[6:7])-1
lat = (lat<<1) + int(i/2)
lon = (lon<<1) + i%2
base1 = base1<<1
else:
lat = lat*10 + int(gridcode[6:7])
lon = lon*10 + int(gridcode[7:8])
base1 = base1*10
if codelen>8:
if gridcode[8:]=='5':
lat = lat>>1
lon = lon>>1
base1 = base1>>1
else:
for i in gridcode[8:]:
i = int(i)-1
lat = (lat<<1) + int(i/2)
lon = (lon<<1) + i%2
base1 = base1<<1
return (lat, lon, base1)
def decode_sw(gridcode, delta=False):
(lat, lon, base1) = _decode_c2i(gridcode)
lat = lat/(base1*1.5)
lon = lon/float(base1) + 100.0
if delta:
return (lat, lon, 1.0/(base1*1.5), 1.0/base1)
else:
return (lat, lon)
def decode(gridcode):
(lat, lon, base1) = _decode_c2i(gridcode)
# center position of the meshcode.
lat = (lat<<1) + 1
lon = (lon<<1) + 1
base1 = base1<<1
return (lat/(base1*1.5), lon/float(base1) + 100.0)
def bbox(gridcode):
(a,b,c,d) = decode_sw(gridcode, True)
return {'w':a, 's':b, 'n':b+d, 'e':a+c}
## short-cut methods
def encodeLv1(lat, lon):
return encode(lat,lon,1)
def encodeLv2(lat, lon):
return encode(lat,lon,8)
def encodeLv3(lat, lon):
return encode(lat,lon,80)
def encodeBase(lat,lon):
return encodeLv3(lat,lon)
def encodeHalf(lat,lon):
return encode(lat,lon,160)
def encodeQuarter(lat,lon):
return encode(lat,lon,320)
def encodeEighth(lat,lon):
return encode(lat,lon,640)
def encodeUni10(lat,lon):
return encodeLv2(lat,lon)
def encodeUni5(lat, lon):
return encode(lat,lon,16)
def encodeUni2(lat, lon):
return encode(lat,lon,40)
def neighbors(gridcode):
(lat,lon,base1)=_decode_c2i(gridcode)
ret = []
for i in ((0,-1),(0,1),(1,-1),(1,0),(1,1),(-1,-1),(-1,0),(-1,1)):
tlat=lat+i[0]
tlon=lon+i[1]
if tlat<0 or tlat>(90*base1):
continue
if tlon<0 or tlon>(100*base1):
continue
ret.append(_encode_i2c(tlat,tlon,base1))
return ret
def expand(gridcode):
ret = neighbors(gridcode)
ret.append(gridcode)
return ret

87
infrastructure/sandbox/Data/lambda/jpiarea.py
View file

@ -1,87 +0,0 @@
# coding: UTF-8
# Coder for Japanese iarea grid code.
# NTT DoCoMo's Open iArea in Japan use a gridcode which is very similar to
# JIS X 0410, but absolutely different in detail.
def _encode_i2c(lat,lon,basebits):
t=[]
for i in range(basebits-3):
t.append((lat&1)*2 + (lon&1))
lat = lat>>1
lon = lon>>1
if basebits>=3:
t.append(lon&7)
t.append(lat&7)
lat = lat>>3
lon = lon>>3
t.append(lon)
t.append(lat)
t.reverse()
return ''.join([str(i) for i in t])
def encode(lat, lon):
if lat<7 or lon<100:
raise Exception('Unsupported location')
basebits = 8
return _encode_i2c(int(lat * (1<<basebits) * 1.5), int((lon-100.0)*(1<<basebits)), basebits)
def _decode_c2i(gridcode):
lat = lon = 0
base = 1
basebits = 0
if len(gridcode)>6:
for i in gridcode[6:]:
lat = (lat<<1) + int(int(i)/2)
lon = (lon<<1) + int(i)%2
base = base<<1
basebits += 1
if len(gridcode)>4:
lat = int(gridcode[4:5])*base + lat
lon = int(gridcode[5:6])*base + lon
base = base<<3
basebits += 3
lat = int(gridcode[0:2])*base + lat
lon = int(gridcode[2:4])*base + lon
return (lat, lon, basebits)
def decode_sw(gridcode, delta=False):
lat, lon, basebits = _decode_c2i(gridcode)
if delta:
return (float(lat)/(1.5*(1<<basebits)), float(lon)/(1<<basebits)+100.0, 1.0/(1.5*(1<<basebits)), 1.0/(1<<basebits))
else:
return (float(lat)/(1.5*(1<<basebits)), float(lon)/(1<<basebits)+100.0)
def decode(gridcode):
lat, lon, basebits = _decode_c2i(gridcode)
return ((lat<<1)+1)/float(3<<basebits), 100.0+((lon<<1)+1)/float(2<<basebits)
def bbox(gridcode):
(a,b,c,d) = decode_sw(gridcode, True)
return {'w':a, 's':b, 'n':b+d, 'e':a+c}
def neighbors(gridcode):
(lat,lon,basebits)=_decode_c2i(gridcode)
ret = []
for i in ((0,-1),(0,1),(1,-1),(1,0),(1,1),(-1,-1),(-1,0),(-1,1)):
tlat=lat+i[0]
tlon=lon+i[1]
if tlat<0 or tlat>(90<<basebits):
continue
if tlon<0 or tlon>(100<<basebits):
continue
ret.append(_encode_i2c(tlat,tlon,basebits))
return ret
def expand(gridcode):
ret = neighbors(gridcode)
ret.append(gridcode)
return ret

120
infrastructure/sandbox/Data/lambda/main.py
View file

@ -1,120 +0,0 @@
'''Ships alb logs from s3 to es'''
import os
import re
import gzip
import logging
import hashlib
import geohash
import urllib.parse
import boto3
from geoip import geolite2
from elasticsearch import Elasticsearch
from elasticsearch.helpers import bulk
from elasticsearch.serializer import JSONSerializer
print('Loading function')
s3 = boto3.client('s3')
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
es_logger = logging.getLogger('elasticsearch')
es_logger.setLevel(logging.DEBUG)
class SetEncoder(JSONSerializer):
def default(self, obj):
if isinstance(obj, frozenset):
return list(obj)
return JSONSerializer.default(self, obj)
def parse_int(thing):
try:
return int(thing)
except:
return None
fields = (
("type", str),
("time", str),
("elb", str),
("client_ip", str),
("client_port", parse_int),
("target_ip", str),
("target_port", parse_int),
("request_processing_time", float),
("target_processing_time", float),
("response_processing_time", float),
("elb_status_code", parse_int),
("target_status_code", str),
("received_bytes", parse_int),
("sent_bytes", parse_int),
("request_verb", str),
("request_url", lambda a: urllib.parse.urlsplit(a)._asdict()),
("request_proto", str),
("user_agent", str),
("ssl_cipher", str),
("ssl_protocol", str),
("target_group_arn", str),
("trace_id", str),
("domain_name", str),
("chosen_cert_arn", str),
("matched_rule_priority", str),
("request_creation_time", str),
("actions_executed", str),
("redirect_url", str),
("lambda_error_reason", str),
("target_port_list", str),
("target_status_code_list", str),
("classification", str),
("classification_reason", str),
)
REGEX = r'([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*):([0-9]*) ([^ ]*)[:-]([0-9]*) ([-.0-9]*) ([-.0-9]*) ([-.0-9]*) (|[-0-9]*) (-|[-0-9]*) ([-0-9]*) ([-0-9]*) \"([^ ]*) (.*) (- |[^ ]*)\" \"([^\"]*)\" ([A-Z0-9-_]+) ([A-Za-z0-9.-]*) ([^ ]*) \"([^\"]*)\" \"([^\"]*)\" \"([^\"]*)\" ([-.0-9]*) ([^ ]*) \"([^\"]*)\" \"([^\"]*)\" \"([^ ]*)\" \"([^\s]+?)\" \"([^\s]+)\" \"([^ ]*)\" \"([^ ]*)\"'
MATCHER = re.compile(REGEX)
AFFINITY = {
'/api/latest/fleet/download_installer/pkg': 'mac',
'/api/latest/fleet/download_installer/msi': 'windows',
'/api/latest/fleet/download_installer/deb': 'linux',
'/api/latest/fleet/download_installer/rpm': 'linux',
}
ENRICHERS = [
lambda a: {'geoip': geolite2.lookup(a['client_ip']).to_dict() if geolite2.lookup(a['client_ip']) is not None else None},
lambda a: {'geohash': geohash.encode(*a['geoip']['location']) if a['geoip'] is not None else None},
lambda a: {'os_affinity': AFFINITY[a['request_url']['path']] if a['request_url']['path'] in AFFINITY else None},
]
def do_file(bucket, key):
'''Generates log lines'''
search = Elasticsearch([os.environ['ES_URL']], serializer=SetEncoder())
out = []
response = s3.get_object(Bucket=bucket, Key=key)
with gzip.GzipFile(fileobj=response["Body"]) as handle:
for line in handle:
line = line.decode('utf8')
match = MATCHER.match(line)
if not match:
raise line
thing = {i[0]: i[1](match.group(n+1)) for n, i in enumerate(fields)}
thing['_index'] = 'sandbox-prod'
thing['_id'] = hashlib.sha256(line.encode('utf8')).hexdigest()
if thing['elb_status_code'] == 200:
for enricher in ENRICHERS:
thing.update(enricher(thing))
out.append(thing)
logger.debug(f"Sending {len(out)} items to {os.environ['ES_URL']}")
bulk(search, out, chunk_size=100)
def lambda_handler(event, _):
'''Main function'''
#print("Received event: " + json.dumps(event, indent=2))
# Get the object from the event and show its content type
logger.debug(event)
bucket = event['Records'][0]['s3']['bucket']['name']
key = urllib.parse.unquote_plus(event['Records'][0]['s3']['object']['key'], encoding='utf-8')
do_file(bucket, key)

1
infrastructure/sandbox/Data/lambda/python_geohash-0.8.5.dist-info/INSTALLER
View file

@ -1 +0,0 @@
pip

11
infrastructure/sandbox/Data/lambda/python_geohash-0.8.5.dist-info/METADATA
View file

@ -1,11 +0,0 @@
Metadata-Version: 2.1
Name: python-geohash
Version: 0.8.5
Summary: Fast, accurate python geohashing library
Home-page: http://code.google.com/p/python-geohash/
Author: Hiroaki Kawai
License: UNKNOWN
Platform: UNKNOWN
UNKNOWN

15
infrastructure/sandbox/Data/lambda/python_geohash-0.8.5.dist-info/RECORD
View file

@ -1,15 +0,0 @@
__pycache__/geohash.cpython-310.pyc,,
__pycache__/jpgrid.cpython-310.pyc,,
__pycache__/jpiarea.cpython-310.pyc,,
__pycache__/quadtree.cpython-310.pyc,,
_geohash.cpython-310-x86_64-linux-gnu.so,sha256=tIe3mA2z1Kp68SvXEJWb2BV6Sjjzwj63rxKNiEaR_QA,96784
geohash.py,sha256=SCeCFfZn8Yd6Wy8vQLwYVGz_Tv-4A9yEjp6dzRhy7BQ,13863
jpgrid.py,sha256=6y0DWfUdCWBD8DP8eESqyog5XpUl8Zsq1AM-7832cgQ,3422
jpiarea.py,sha256=xla1SF1M5F3-YRZt-7dFJcH8yFOPh9dgTY8pSLu8Z50,2176
python_geohash-0.8.5.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
python_geohash-0.8.5.dist-info/METADATA,sha256=nhqCwV4ep_ybb1FSeMTbe_BmAmowfQX6q6pgzZcIpy8,227
python_geohash-0.8.5.dist-info/RECORD,,
python_geohash-0.8.5.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
python_geohash-0.8.5.dist-info/WHEEL,sha256=nTy_Z8ivGEB6qFwVLg7_h6rq0Jt0JU5Pz2cL2_FjSMQ,105
python_geohash-0.8.5.dist-info/top_level.txt,sha256=rENHxyvjpwkQAvPvVXCKloPkH3xf5qSuDvGZoOwHqoQ,41
quadtree.py,sha256=py6QoRVa8uZItbrC7M2AVp-QPJnS9VD-Ebq-2Wl2nxw,2696

0
infrastructure/sandbox/Data/lambda/python_geohash-0.8.5.dist-info/REQUESTED
View file

5
infrastructure/sandbox/Data/lambda/python_geohash-0.8.5.dist-info/WHEEL
View file

@ -1,5 +0,0 @@
Wheel-Version: 1.0
Generator: bdist_wheel (0.37.1)
Root-Is-Purelib: false
Tag: cp310-cp310-linux_x86_64

Some files were not shown because too many files have changed in this diff Show more