commit 22e69251d9b5cd2611abf77ef7352abfa4d409d7 Author: Théophile Diot Date: Fri Jun 30 15:37:37 2023 -0400 Squashed 'src/deps/src/ngx_brotli/' content from commit 6e975bcb0 git-subtree-dir: src/deps/src/ngx_brotli git-subtree-split: 6e975bcb015f62e1f303054897783355e2a877dc diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000..694f08884 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "deps/brotli"] + path = deps/brotli + url = https://github.com/google/brotli.git diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 000000000..57d2b8a79 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,46 @@ +# required for http2 support in curl. +dist: bionic +language: c +sudo: false +matrix: + include: + # unfortunately, gcc-4.9 is dropped in bionic + - os: linux + addons: + apt: + sources: + - ubuntu-toolchain-r-test + packages: + - g++-5 + env: + - MATRIX_EVAL="CC=gcc-5 && CXX=g++-5" + + - os: linux + addons: + apt: + sources: + - ubuntu-toolchain-r-test + packages: + - g++-6 + env: + - MATRIX_EVAL="CC=gcc-6 && CXX=g++-6" + - os: linux + addons: + apt: + sources: + - ubuntu-toolchain-r-test + packages: + - g++-7 + env: + - MATRIX_EVAL="CC=gcc-7 && CXX=g++-7" + + +script: +- script/.travis-compile.sh +- script/.travis-before-test.sh +- script/.travis-test.sh +after_success: +- killall nginx +after_failure: +- killall nginx + diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 000000000..a9eabff7f --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,27 @@ +# Contributing + +Want to contribute? Great! First, read this page (including the small print at the end). + +### Before you contribute +Before we can use your code, you must sign the +[Google Individual Contributor License Agreement](https://cla.developers.google.com/about/google-individual) +(CLA), which you can do online. The CLA is necessary mainly because you own the +copyright to your changes, even after your contribution becomes part of our +codebase, so we need your permission to use and distribute your code. We also +need to be sure of various other things—for instance that you'll tell us if you +know that your code infringes on other people's patents. You don't have to sign +the CLA until after you've submitted your code for review and a member has +approved it, but you must do it before we can put your code into our codebase. +Before you start working on a larger contribution, you should get in touch with +us first through the issue tracker with your idea so that we can help out and +possibly guide you. Coordinating up front makes it much easier to avoid +frustration later on. + +### Code reviews +All submissions, including submissions by project members, require review. We +use Github pull requests for this purpose. + +### The small print +Contributions made by corporations are covered by a different agreement than +the one above, the +[Software Grant and Corporate Contributor License Agreement](https://cla.developers.google.com/about/google-corporate). diff --git a/LICENSE b/LICENSE new file mode 100644 index 000000000..db3aca78e --- /dev/null +++ b/LICENSE @@ -0,0 +1,27 @@ +/* + * Copyright (C) 2002-2015 Igor Sysoev + * Copyright (C) 2011-2015 Nginx, Inc. + * Copyright (C) 2015-2019 Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ diff --git a/README.md b/README.md new file mode 100644 index 000000000..80428e95a --- /dev/null +++ b/README.md @@ -0,0 +1,177 @@ +# ngx_brotli + +Brotli is a generic-purpose lossless compression algorithm that compresses data +using a combination of a modern variant of the LZ77 algorithm, Huffman coding +and 2nd order context modeling, with a compression ratio comparable to the best +currently available general-purpose compression methods. It is similar in speed +with deflate but offers more dense compression. + +ngx_brotli is a set of two nginx modules: + +- ngx_brotli filter module - used to compress responses on-the-fly, +- ngx_brotli static module - used to serve pre-compressed files. + +[![TravisCI Build Status](https://travis-ci.org/google/ngx_brotli.svg?branch=master)](https://travis-ci.org/google/ngx_brotli) + +## Table of Contents + +- [Status](#status) +- [Installation](#installation) +- [Configuration directives](#configuration-directives) + - [`brotli_static`](#brotli_static) + - [`brotli`](#brotli) + - [`brotli_types`](#brotli_types) + - [`brotli_buffers`](#brotli_buffers) + - [`brotli_comp_level`](#brotli_comp_level) + - [`brotli_window`](#brotli_window) + - [`brotli_min_length`](#brotli_min_length) +- [Variables](#variables) + - [`$brotli_ratio`](#brotli_ratio) +- [Sample configuration](#sample-configuration) +- [Contributing](#contributing) +- [License](#license) + +## Status + +Both Brotli library and nginx module are under active development. + +## Installation + +### Dynamically loaded + + $ cd nginx-1.x.x + $ ./configure --with-compat --add-dynamic-module=/path/to/ngx_brotli + $ make modules + +You will need to use **exactly** the same `./configure` arguments as your Nginx configuration and append `--with-compat --add-dynamic-module=/path/to/ngx_brotli` to the end, otherwise you will get a "module is not binary compatible" error on startup. You can run `nginx -V` to get the configuration arguments for your Nginx installation. + +`make modules` will result in `ngx_http_brotli_filter_module.so` and `ngx_http_brotli_static_module.so` in the `objs` directory. Copy these to `/usr/lib/nginx/modules/` then add the `load_module` directives to `nginx.conf` (above the http block): +```nginx +load_module modules/ngx_http_brotli_filter_module.so; +load_module modules/ngx_http_brotli_static_module.so; +``` + +### Statically compiled + + $ cd nginx-1.x.x + $ ./configure --add-module=/path/to/ngx_brotli + $ make && make install + +This will compile the module directly into Nginx. + +## Configuration directives + +### `brotli_static` + +- **syntax**: `brotli_static on|off|always` +- **default**: `off` +- **context**: `http`, `server`, `location` + +Enables or disables checking of the existence of pre-compressed files with`.br` +extension. With the `always` value, pre-compressed file is used in all cases, +without checking if the client supports it. + +### `brotli` + +- **syntax**: `brotli on|off` +- **default**: `off` +- **context**: `http`, `server`, `location`, `if` + +Enables or disables on-the-fly compression of responses. + +### `brotli_types` + +- **syntax**: `brotli_types [..]` +- **default**: `text/html` +- **context**: `http`, `server`, `location` + +Enables on-the-fly compression of responses for the specified MIME types +in addition to `text/html`. The special value `*` matches any MIME type. +Responses with the `text/html` MIME type are always compressed. + +### `brotli_buffers` + +- **syntax**: `brotli_buffers ` +- **default**: `32 4k|16 8k` +- **context**: `http`, `server`, `location` + +**Deprecated**, ignored. + +### `brotli_comp_level` + +- **syntax**: `brotli_comp_level ` +- **default**: `6` +- **context**: `http`, `server`, `location` + +Sets on-the-fly compression Brotli quality (compression) `level`. +Acceptable values are in the range from `0` to `11`. + +### `brotli_window` + +- **syntax**: `brotli_window ` +- **default**: `512k` +- **context**: `http`, `server`, `location` + +Sets Brotli window `size`. Acceptable values are `1k`, `2k`, `4k`, `8k`, `16k`, +`32k`, `64k`, `128k`, `256k`, `512k`, `1m`, `2m`, `4m`, `8m` and `16m`. + +### `brotli_min_length` + +- **syntax**: `brotli_min_length ` +- **default**: `20` +- **context**: `http`, `server`, `location` + +Sets the minimum `length` of a response that will be compressed. +The length is determined only from the `Content-Length` response header field. + +## Variables + +### `$brotli_ratio` + +Achieved compression ratio, computed as the ratio between the original +and compressed response sizes. + +## Sample configuration + +``` +brotli on; +brotli_comp_level 6; +brotli_static on; +brotli_types application/atom+xml application/javascript application/json application/rss+xml + application/vnd.ms-fontobject application/x-font-opentype application/x-font-truetype + application/x-font-ttf application/x-javascript application/xhtml+xml application/xml + font/eot font/opentype font/otf font/truetype image/svg+xml image/vnd.microsoft.icon + image/x-icon image/x-win-bitmap text/css text/javascript text/plain text/xml; +``` + +## Contributing + +See [Contributing](CONTRIBUTING.md). + +## License + + Copyright (C) 2002-2015 Igor Sysoev + Copyright (C) 2011-2015 Nginx, Inc. + Copyright (C) 2015 Google Inc. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + SUCH DAMAGE. diff --git a/config b/config new file mode 100644 index 000000000..42162a6e8 --- /dev/null +++ b/config @@ -0,0 +1,34 @@ +# Copyright (C) 2019 Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. + +# Make sure the module knows it is a submodule. +ngx_addon_name=ngx_brotli +. $ngx_addon_dir/filter/config + +# Make sure the module knows it is a submodule. +ngx_addon_name=ngx_brotli +. $ngx_addon_dir/static/config + +# The final name for reporting. +ngx_addon_name=ngx_brotli diff --git a/deps/brotli b/deps/brotli new file mode 160000 index 000000000..f4153a09f --- /dev/null +++ b/deps/brotli @@ -0,0 +1 @@ +Subproject commit f4153a09f87cbb9c826d8fc12c74642bb2d879ea diff --git a/filter/config b/filter/config new file mode 100644 index 000000000..9c27fc777 --- /dev/null +++ b/filter/config @@ -0,0 +1,132 @@ +# Copyright (C) 2015-2016 Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. + +if [ "$ngx_addon_name" = "ngx_brotli" ]; then + BROTLI_MODULE_SRC_DIR="$ngx_addon_dir/filter" +else + BROTLI_MODULE_SRC_DIR="$ngx_addon_dir" +fi + +ngx_addon_name=ngx_brotli_filter + +if [ -z "$ngx_module_link" ]; then +cat << END + +$0: error: Brotli module requires recent version of NGINX (1.9.11+). + +END + exit 1 +fi + +ngx_module_type=HTTP_FILTER +ngx_module_name=ngx_http_brotli_filter_module + +brotli="$ngx_addon_dir/deps/brotli/c" +if [ ! -f "$brotli/include/brotli/encode.h" ]; then + brotli="/usr/local" +fi +if [ ! -f "$brotli/include/brotli/encode.h" ]; then + brotli="/usr" +fi +if [ ! -f "$brotli/include/brotli/encode.h" ]; then +cat << END + +$0: error: \ +Brotli library is missing from the $brotli directory. + +Please make sure that the git submodule has been checked out: + + cd $ngx_addon_dir && git submodule update --init && cd $PWD + +END + exit 1 +fi + +BROTLI_LISTS_FILE="$brotli/../scripts/sources.lst" + +if [ -f "$BROTLI_LISTS_FILE" ]; then + +BROTLI_LISTS=`cat "$BROTLI_LISTS_FILE" | grep -v "#" | tr '\n' '#' | \ + sed 's/\\\\#//g' | tr -s ' ' '+' | tr -s '#' ' ' | \ + sed 's/+c/+$brotli/g' | sed 's/+=+/=/g'` +for ITEM in ${BROTLI_LISTS}; do + VAR=`echo $ITEM | sed 's/=.*//'` + VAL=`echo $ITEM | sed 's/.*=//' | tr '+' ' '` + eval ${VAR}=\"$VAL\" +done + +else # BROTLI_LISTS_FILE + +BROTLI_ENC_H="$brotli/include/brotli/encode.h \ + $brotli/include/brotli/port.h \ + $brotli/include/brotli/types.h" +BROTLI_ENC_LIB="-lbrotlienc" + +fi + +ngx_module_incs="$brotli/include" +ngx_module_deps="$BROTLI_COMMON_H $BROTLI_ENC_H" +ngx_module_srcs="$BROTLI_COMMON_C $BROTLI_ENC_C \ + $BROTLI_MODULE_SRC_DIR/ngx_http_brotli_filter_module.c" +ngx_module_libs="$BROTLI_ENC_LIB -lm" +ngx_module_order="$ngx_module_name \ + ngx_pagespeed \ + ngx_http_postpone_filter_module \ + ngx_http_ssi_filter_module \ + ngx_http_charset_filter_module \ + ngx_http_xslt_filter_module \ + ngx_http_image_filter_module \ + ngx_http_sub_filter_module \ + ngx_http_addition_filter_module \ + ngx_http_gunzip_filter_module \ + ngx_http_userid_filter_module \ + ngx_http_headers_filter_module \ + ngx_http_copy_filter_module \ + ngx_http_range_body_filter_module \ + ngx_http_not_modified_filter_module \ + ngx_http_slice_filter_module" + +. auto/module + +if [ "$ngx_module_link" != DYNAMIC ]; then + # ngx_module_order doesn't work with static modules, + # so we must re-order filters here. + + if [ "$HTTP_GZIP" = YES ]; then + next=ngx_http_gzip_filter_module + elif echo $HTTP_FILTER_MODULES | grep pagespeed_etag_filter >/dev/null; then + next=ngx_pagespeed_etag_filter + else + next=ngx_http_range_header_filter_module + fi + + HTTP_FILTER_MODULES=`echo $HTTP_FILTER_MODULES \ + | sed "s/$ngx_module_name//" \ + | sed "s/$next/$next $ngx_module_name/"` +fi + +CFLAGS="$CFLAGS -Wno-deprecated-declarations" + +have=NGX_HTTP_BROTLI_FILTER . auto/have +have=NGX_HTTP_BROTLI_FILTER_MODULE . auto/have # deprecated diff --git a/filter/ngx_http_brotli_filter_module.c b/filter/ngx_http_brotli_filter_module.c new file mode 100644 index 000000000..592b767a6 --- /dev/null +++ b/filter/ngx_http_brotli_filter_module.c @@ -0,0 +1,770 @@ +/* + * Copyright (C) Igor Sysoev + * Copyright (C) Nginx, Inc. + * Copyright (C) Google Inc. + */ + +#include +#include +#include + +#if (NGX_HAVE_BROTLI_ENC_ENCODE_H) +#include +#else +#include +#endif + +/* Brotli and GZip modules never stack, i.e. when one of them sets + "Content-Encoding" the other becomes a pass-through filter. Consequently, + it is almost legal to reuse this "buffered" bit. + IIUC, buffered == some data passed to filter has not been pushed further. */ +#define NGX_HTTP_BROTLI_BUFFERED NGX_HTTP_GZIP_BUFFERED + +/* Module configuration. */ +typedef struct { + ngx_flag_t enable; + + /* Supported MIME types. */ + ngx_hash_t types; + ngx_array_t* types_keys; + + /* Minimal required length for compression (if known). */ + ssize_t min_length; + + ngx_bufs_t deprecated_unused_bufs; + + /* Brotli encoder parameter: quality */ + ngx_int_t quality; + + /* Brotli encoder parameter: (max) lg_win */ + size_t lg_win; +} ngx_http_brotli_conf_t; + +/* Instance context. */ +typedef struct { + /* Brotli encoder instance. */ + BrotliEncoderState* encoder; + + /* Payload length; -1, if unknown. */ + off_t content_length; + + /* (uncompressed) bytes pushed to encoder. */ + size_t bytes_in; + /* (compressed) bytes pulled from encoder. */ + size_t bytes_out; + + /* Input buffer chain. */ + ngx_chain_t* in; + + /* Output chain. */ + ngx_chain_t* out_chain; + + /* Output buffer. */ + ngx_buf_t* out_buf; + + /* Various state flags. */ + + /* 1 if encoder is initialized, output chain and buffer are allocated. */ + unsigned initialized : 1; + /* 1 if compression is finished / failed. */ + unsigned closed : 1; + /* 1 if compression is finished. */ + unsigned success : 1; + + /* 1 if out_chain is ready to be committed, 0 otherwise. */ + unsigned output_ready : 1; + /* 1 if output buffer is committed to the next filter and not yet fully used. + 0 otherwise. */ + unsigned output_busy : 1; + + unsigned end_of_input : 1; + unsigned end_of_block : 1; + + ngx_http_request_t* request; +} ngx_http_brotli_ctx_t; + +/* Forward declarations. */ + +/* Initializes encoder, output chain and buffer, if necessary. Returns NGX_OK + if encoder is successfully initialized (have been already initialized), + and requires objects are allocated. Returns NGX_ERROR otherwise. */ +static ngx_int_t ngx_http_brotli_filter_ensure_stream_initialized( + ngx_http_request_t* r, ngx_http_brotli_ctx_t* ctx); +/* Marks instance as closed and performs cleanup. */ +static void ngx_http_brotli_filter_close(ngx_http_brotli_ctx_t* ctx); + +static void* ngx_http_brotli_filter_alloc(void* opaque, size_t size); +static void ngx_http_brotli_filter_free(void* opaque, void* address); + +static ngx_int_t ngx_http_brotli_check_request(ngx_http_request_t* r); + +static ngx_int_t ngx_http_brotli_add_variables(ngx_conf_t* cf); +static ngx_int_t ngx_http_brotli_ratio_variable(ngx_http_request_t* r, + ngx_http_variable_value_t* v, + uintptr_t data); + +static void* ngx_http_brotli_create_conf(ngx_conf_t* cf); +static char* ngx_http_brotli_merge_conf(ngx_conf_t* cf, void* parent, + void* child); +static ngx_int_t ngx_http_brotli_filter_init(ngx_conf_t* cf); + +static char* ngx_http_brotli_parse_wbits(ngx_conf_t* cf, void* post, + void* data); + +/* Configuration literals. */ + +static ngx_conf_num_bounds_t ngx_http_brotli_comp_level_bounds = { + ngx_conf_check_num_bounds, BROTLI_MIN_QUALITY, BROTLI_MAX_QUALITY}; + +static ngx_conf_post_handler_pt ngx_http_brotli_parse_wbits_p = + ngx_http_brotli_parse_wbits; + +static ngx_command_t ngx_http_brotli_filter_commands[] = { + {ngx_string("brotli"), + NGX_HTTP_MAIN_CONF | NGX_HTTP_SRV_CONF | NGX_HTTP_LOC_CONF | + NGX_HTTP_LIF_CONF | NGX_CONF_FLAG, + ngx_conf_set_flag_slot, NGX_HTTP_LOC_CONF_OFFSET, + offsetof(ngx_http_brotli_conf_t, enable), NULL}, + + /* Deprecated, unused. */ + {ngx_string("brotli_buffers"), + NGX_HTTP_MAIN_CONF | NGX_HTTP_SRV_CONF | NGX_HTTP_LOC_CONF | + NGX_CONF_TAKE2, + ngx_conf_set_bufs_slot, NGX_HTTP_LOC_CONF_OFFSET, + offsetof(ngx_http_brotli_conf_t, deprecated_unused_bufs), NULL}, + + {ngx_string("brotli_types"), + NGX_HTTP_MAIN_CONF | NGX_HTTP_SRV_CONF | NGX_HTTP_LOC_CONF | + NGX_CONF_1MORE, + ngx_http_types_slot, NGX_HTTP_LOC_CONF_OFFSET, + offsetof(ngx_http_brotli_conf_t, types_keys), + &ngx_http_html_default_types[0]}, + + {ngx_string("brotli_comp_level"), + NGX_HTTP_MAIN_CONF | NGX_HTTP_SRV_CONF | NGX_HTTP_LOC_CONF | + NGX_CONF_TAKE1, + ngx_conf_set_num_slot, NGX_HTTP_LOC_CONF_OFFSET, + offsetof(ngx_http_brotli_conf_t, quality), + &ngx_http_brotli_comp_level_bounds}, + + {ngx_string("brotli_window"), + NGX_HTTP_MAIN_CONF | NGX_HTTP_SRV_CONF | NGX_HTTP_LOC_CONF | + NGX_CONF_TAKE1, + ngx_conf_set_size_slot, NGX_HTTP_LOC_CONF_OFFSET, + offsetof(ngx_http_brotli_conf_t, lg_win), &ngx_http_brotli_parse_wbits_p}, + + {ngx_string("brotli_min_length"), + NGX_HTTP_MAIN_CONF | NGX_HTTP_SRV_CONF | NGX_HTTP_LOC_CONF | + NGX_CONF_TAKE1, + ngx_conf_set_size_slot, NGX_HTTP_LOC_CONF_OFFSET, + offsetof(ngx_http_brotli_conf_t, min_length), NULL}, + + ngx_null_command}; + +/* Module context hooks. */ +static ngx_http_module_t ngx_http_brotli_filter_module_ctx = { + ngx_http_brotli_add_variables, /* pre-configuration */ + ngx_http_brotli_filter_init, /* post-configuration */ + + NULL, /* create main configuration */ + NULL, /* init main configuration */ + + NULL, /* create server configuration */ + NULL, /* merge server configuration */ + + ngx_http_brotli_create_conf, /* create location configuration */ + ngx_http_brotli_merge_conf /* merge location configuration */ +}; + +/* Module descriptor. */ +ngx_module_t ngx_http_brotli_filter_module = { + NGX_MODULE_V1, + &ngx_http_brotli_filter_module_ctx, /* module context */ + ngx_http_brotli_filter_commands, /* module directives */ + NGX_HTTP_MODULE, /* module type */ + NULL, /* init master */ + NULL, /* init module */ + NULL, /* init process */ + NULL, /* init thread */ + NULL, /* exit thread */ + NULL, /* exit process */ + NULL, /* exit master */ + NGX_MODULE_V1_PADDING}; + +/* Variable names. */ +static ngx_str_t ngx_http_brotli_ratio = ngx_string("brotli_ratio"); + +/* Next filter in the filter chain. */ +static ngx_http_output_header_filter_pt ngx_http_next_header_filter; +static ngx_http_output_body_filter_pt ngx_http_next_body_filter; + +static /* const */ char kEncoding[] = "br"; +static const size_t kEncodingLen = 2; + +static ngx_int_t check_accept_encoding(ngx_http_request_t* req) { + ngx_table_elt_t* accept_encoding_entry; + ngx_str_t* accept_encoding; + u_char* cursor; + u_char* end; + u_char before; + u_char after; + + accept_encoding_entry = req->headers_in.accept_encoding; + if (accept_encoding_entry == NULL) return NGX_DECLINED; + accept_encoding = &accept_encoding_entry->value; + + cursor = accept_encoding->data; + end = cursor + accept_encoding->len; + while (1) { + u_char digit; + /* It would be an idiotic idea to rely on compiler to produce performant + binary, that is why we just do -1 at every call site. */ + cursor = ngx_strcasestrn(cursor, kEncoding, kEncodingLen - 1); + if (cursor == NULL) return NGX_DECLINED; + before = (cursor == accept_encoding->data) ? ' ' : cursor[-1]; + cursor += kEncodingLen; + after = (cursor >= end) ? ' ' : *cursor; + if (before != ',' && before != ' ') continue; + if (after != ',' && after != ' ' && after != ';') continue; + + /* Check for ";q=0[.[0[0[0]]]]" */ + while (*cursor == ' ') cursor++; + if (*(cursor++) != ';') break; + while (*cursor == ' ') cursor++; + if (*(cursor++) != 'q') break; + while (*cursor == ' ') cursor++; + if (*(cursor++) != '=') break; + while (*cursor == ' ') cursor++; + if (*(cursor++) != '0') break; + if (*(cursor++) != '.') return NGX_DECLINED; /* ;q=0, */ + digit = *(cursor++); + if (digit < '0' || digit > '9') return NGX_DECLINED; /* ;q=0., */ + if (digit > '0') break; + digit = *(cursor++); + if (digit < '0' || digit > '9') return NGX_DECLINED; /* ;q=0.0, */ + if (digit > '0') break; + digit = *(cursor++); + if (digit < '0' || digit > '9') return NGX_DECLINED; /* ;q=0.00, */ + if (digit > '0') break; + return NGX_DECLINED; /* ;q=0.000 */ + } + return NGX_OK; +} + +/* Process headers and decide if request is eligible for brotli compression. */ +static ngx_int_t ngx_http_brotli_header_filter(ngx_http_request_t* r) { + ngx_table_elt_t* h; + ngx_http_brotli_ctx_t* ctx; + ngx_http_brotli_conf_t* conf; + + conf = ngx_http_get_module_loc_conf(r, ngx_http_brotli_filter_module); + + /* Filter only if enabled. */ + if (!conf->enable) { + return ngx_http_next_header_filter(r); + } + + /* Only compress OK / forbidden / not found responses. */ + if (r->headers_out.status != NGX_HTTP_OK && + r->headers_out.status != NGX_HTTP_FORBIDDEN && + r->headers_out.status != NGX_HTTP_NOT_FOUND) { + return ngx_http_next_header_filter(r); + } + + /* Bypass "header only" responses. */ + if (r->header_only) { + return ngx_http_next_header_filter(r); + } + + /* Bypass already compressed responses. */ + if (r->headers_out.content_encoding && + r->headers_out.content_encoding->value.len) { + return ngx_http_next_header_filter(r); + } + + /* If response size is known, do not compress tiny responses. */ + if (r->headers_out.content_length_n != -1 && + r->headers_out.content_length_n < conf->min_length) { + return ngx_http_next_header_filter(r); + } + + /* Compress only certain MIME-typed responses. */ + if (ngx_http_test_content_type(r, &conf->types) == NULL) { + return ngx_http_next_header_filter(r); + } + + r->gzip_vary = 1; + + /* Check if client support brotli encoding. */ + if (ngx_http_brotli_check_request(r) != NGX_OK) { + return ngx_http_next_header_filter(r); + } + + /* Prepare instance context. */ + ctx = ngx_pcalloc(r->pool, sizeof(ngx_http_brotli_ctx_t)); + if (ctx == NULL) { + return NGX_ERROR; + } + ctx->request = r; + ctx->content_length = r->headers_out.content_length_n; + ngx_http_set_ctx(r, ctx, ngx_http_brotli_filter_module); + + /* Prepare response headers, so that following filters in the chain will + notice that response body is compressed. */ + h = ngx_list_push(&r->headers_out.headers); + if (h == NULL) { + return NGX_ERROR; + } + + h->hash = 1; + ngx_str_set(&h->key, "Content-Encoding"); + ngx_str_set(&h->value, "br"); + r->headers_out.content_encoding = h; + + r->main_filter_need_in_memory = 1; + + ngx_http_clear_content_length(r); + ngx_http_clear_accept_ranges(r); + ngx_http_weak_etag(r); + + return ngx_http_next_header_filter(r); +} + +/* Response body filtration (compression). */ +static ngx_int_t ngx_http_brotli_body_filter(ngx_http_request_t* r, + ngx_chain_t* in) { + int rc; + ngx_http_brotli_ctx_t* ctx; + size_t available_output; + ptrdiff_t available_busy_output; + size_t input_size; + size_t available_input; + const uint8_t* next_input_byte; + size_t consumed_input; + BROTLI_BOOL ok; + u_char* out; + ngx_chain_t* link; + + ctx = ngx_http_get_module_ctx(r, ngx_http_brotli_filter_module); + + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "http brotli filter"); + + if (ctx == NULL || ctx->closed || r->header_only) { + return ngx_http_next_body_filter(r, in); + } + + if (ngx_http_brotli_filter_ensure_stream_initialized(r, ctx) != NGX_OK) { + ngx_http_brotli_filter_close(ctx); + return NGX_ERROR; + } + + /* If more input is provided - append it to our input chain. */ + if (in) { + if (ngx_chain_add_copy(r->pool, &ctx->in, in) != NGX_OK) { + ngx_http_brotli_filter_close(ctx); + return NGX_ERROR; + } + r->connection->buffered |= NGX_HTTP_BROTLI_BUFFERED; + } + + /* Main loop: + - if output is not yet consumed - stop; encoder should not be touched, + until all the output is consumed + - if encoder has output - wrap it and send to consumer + - if encoder is finished (and all output is consumed) - stop + - if there is more input - push it to encoder */ + for (;;) { + if (ctx->output_busy || ctx->output_ready) { + if (ctx->output_busy) { + available_busy_output = ngx_buf_size(ctx->out_buf); + } else { + available_busy_output = 0; + } + + rc = ngx_http_next_body_filter(r, + ctx->output_ready ? ctx->out_chain : NULL); + if (ctx->output_ready) { + ctx->output_ready = 0; + ctx->output_busy = 1; + } + if (ngx_buf_size(ctx->out_buf) == 0) { + ctx->output_busy = 0; + } + if (rc == NGX_OK) { + if (ctx->output_busy && + available_busy_output == ngx_buf_size(ctx->out_buf)) { + r->connection->buffered |= NGX_HTTP_BROTLI_BUFFERED; + return NGX_AGAIN; + } + continue; + } else if (rc == NGX_AGAIN) { + if (ctx->output_busy) { + /* Can't continue compression, let the outer filer decide. */ + if (ctx->in != NULL) { + r->connection->buffered |= NGX_HTTP_BROTLI_BUFFERED; + } + return NGX_AGAIN; + } else { + /* Inner filter has given up, but we can continue processing. */ + continue; + } + } else { + ngx_http_brotli_filter_close(ctx); + return NGX_ERROR; + } + } + + if (BrotliEncoderHasMoreOutput(ctx->encoder)) { + available_output = 0; + out = (u_char*)BrotliEncoderTakeOutput(ctx->encoder, &available_output); + if (out == NULL || available_output == 0) { + ngx_http_brotli_filter_close(ctx); + return NGX_ERROR; + } + ctx->out_buf->start = out; + ctx->out_buf->pos = out; + ctx->out_buf->last = out + available_output; + ctx->out_buf->end = out + available_output; + ctx->bytes_out += available_output; + ctx->out_buf->last_buf = 0; + ctx->out_buf->flush = 0; + if (ctx->end_of_input && BrotliEncoderIsFinished(ctx->encoder)) { + ctx->out_buf->last_buf = 1; + r->connection->buffered &= ~NGX_HTTP_BROTLI_BUFFERED; + } else if (ctx->end_of_block) { + ctx->out_buf->flush = 1; + r->connection->buffered &= ~NGX_HTTP_BROTLI_BUFFERED; + } + ctx->end_of_block = 0; + ctx->output_ready = 1; + ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "brotli out: %p, size:%uz", ctx->out_buf, + ngx_buf_size(ctx->out_buf)); + continue; + } + + if (BrotliEncoderIsFinished(ctx->encoder)) { + ctx->success = 1; + r->connection->buffered &= ~NGX_HTTP_BROTLI_BUFFERED; + ngx_http_brotli_filter_close(ctx); + return NGX_OK; + } + + if (ctx->end_of_input) { + // Ask the encoder to dump the leftover. + available_input = 0; + available_output = 0; + ok = BrotliEncoderCompressStream(ctx->encoder, BROTLI_OPERATION_FINISH, + &available_input, NULL, + &available_output, NULL, NULL); + r->connection->buffered |= NGX_HTTP_BROTLI_BUFFERED; + if (!ok) { + ngx_http_brotli_filter_close(ctx); + return NGX_ERROR; + } + continue; + } + + if (ctx->in == NULL) { + return NGX_OK; + } + + /* TODO: coalesce tiny inputs, if they are not last/flush. */ + input_size = ngx_buf_size(ctx->in->buf); + if (input_size == 0) { + if (!ctx->in->buf->last_buf && !ctx->in->buf->flush) { + link = ctx->in; + ctx->in = ctx->in->next; + ngx_free_chain(r->pool, link); + continue; + } + } + + available_input = input_size; + next_input_byte = (const uint8_t*)ctx->in->buf->pos; + available_output = 0; + ok = BrotliEncoderCompressStream( + ctx->encoder, + ctx->in->buf->last_buf ? BROTLI_OPERATION_FINISH + : ctx->in->buf->flush ? BROTLI_OPERATION_FLUSH + : BROTLI_OPERATION_PROCESS, + &available_input, &next_input_byte, &available_output, NULL, NULL); + r->connection->buffered |= NGX_HTTP_BROTLI_BUFFERED; + if (!ok) { + ngx_http_brotli_filter_close(ctx); + return NGX_ERROR; + } + + consumed_input = input_size - available_input; + ctx->bytes_in += consumed_input; + ctx->in->buf->pos += consumed_input; + + if (consumed_input == input_size) { + if (ctx->in->buf->last_buf) { + ctx->end_of_input = 1; + } else if (ctx->in->buf->flush) { + ctx->end_of_block = 1; + } + link = ctx->in; + ctx->in = ctx->in->next; + ngx_free_chain(r->pool, link); + continue; + } + + /* Should never happen, just to make sure we don't enter infinite loop. */ + if (consumed_input == 0) { + ngx_http_brotli_filter_close(ctx); + return NGX_ERROR; + } + } + + /* unreachable */ + ngx_http_brotli_filter_close(ctx); + return NGX_ERROR; +} + +static ngx_int_t ngx_http_brotli_filter_ensure_stream_initialized( + ngx_http_request_t* r, ngx_http_brotli_ctx_t* ctx) { + ngx_http_brotli_conf_t* conf; + BROTLI_BOOL ok; + size_t wbits; + + if (ctx->initialized) { + return NGX_OK; + } + ctx->initialized = 1; + + conf = ngx_http_get_module_loc_conf(r, ngx_http_brotli_filter_module); + + /* Tune lg_win, if size is known. */ + if (ctx->content_length > 0) { + wbits = BROTLI_MIN_WINDOW_BITS; + while ((wbits < conf->lg_win) && (ctx->content_length > (1 << wbits))) { + wbits++; + } + } else { + wbits = conf->lg_win; + } + + ctx->encoder = BrotliEncoderCreateInstance( + ngx_http_brotli_filter_alloc, ngx_http_brotli_filter_free, r->pool); + if (ctx->encoder == NULL) { + ngx_log_error(NGX_LOG_ALERT, r->connection->log, 0, + "OOM / BrotliEncoderCreateInstance"); + return NGX_ERROR; + } + + ok = BrotliEncoderSetParameter(ctx->encoder, BROTLI_PARAM_QUALITY, + (uint32_t)conf->quality); + if (!ok) { + ngx_log_error(NGX_LOG_ALERT, r->connection->log, 0, + "BrotliEncoderSetParameter(QUALITY, %uD) failed", + (uint32_t)conf->quality); + return NGX_ERROR; + } + + ok = BrotliEncoderSetParameter(ctx->encoder, BROTLI_PARAM_LGWIN, + (uint32_t)wbits); + if (!ok) { + ngx_log_error(NGX_LOG_ALERT, r->connection->log, 0, + "BrotliEncoderSetParameter(LGWIN, %uD) failed", + (uint32_t)wbits); + return NGX_ERROR; + } + + ctx->out_buf = ngx_calloc_buf(r->pool); + if (ctx->out_buf == NULL) { + return NGX_ERROR; + } + ctx->out_buf->temporary = 1; + + ctx->out_chain = ngx_alloc_chain_link(r->pool); + if (ctx->out_chain == NULL) { + return NGX_ERROR; + } + ctx->out_chain->buf = ctx->out_buf; + ctx->out_chain->next = NULL; + + ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "brotli encoder initialized: lvl:%i win:%d", conf->quality, + (1 << wbits)); + + return NGX_OK; +} + +static void* ngx_http_brotli_filter_alloc(void* opaque, size_t size) { + ngx_pool_t* pool = opaque; + void* p; + + p = ngx_palloc(pool, size); + +#if (NGX_DEBUG) + ngx_log_debug2(NGX_LOG_DEBUG_HTTP, pool->log, 0, "brotli alloc: %p, size:%uz", + p, size); +#endif + + return p; +} + +static void ngx_http_brotli_filter_free(void* opaque, void* address) { + ngx_pool_t* pool = opaque; + +#if (NGX_DEBUG) + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pool->log, 0, "brotli free: %p", address); +#endif + + ngx_pfree(pool, address); +} + +static void ngx_http_brotli_filter_close(ngx_http_brotli_ctx_t* ctx) { + ctx->closed = 1; + if (ctx->encoder) { + BrotliEncoderDestroyInstance(ctx->encoder); + ctx->encoder = NULL; + } + if (ctx->out_chain) { + ngx_free_chain(ctx->request->pool, ctx->out_chain); + ctx->out_chain = NULL; + } + if (ctx->out_buf) { + ngx_pfree(ctx->request->pool, ctx->out_buf); + ctx->out_buf = NULL; + } +} + +static ngx_int_t ngx_http_brotli_check_request(ngx_http_request_t* req) { + if (req != req->main) return NGX_DECLINED; + if (check_accept_encoding(req) != NGX_OK) return NGX_DECLINED; + req->gzip_tested = 1; + req->gzip_ok = 0; + return NGX_OK; +} + +static ngx_int_t ngx_http_brotli_add_variables(ngx_conf_t* cf) { + ngx_http_variable_t* var; + + var = ngx_http_add_variable(cf, &ngx_http_brotli_ratio, 0); + if (var == NULL) { + return NGX_ERROR; + } + + var->get_handler = ngx_http_brotli_ratio_variable; + + return NGX_OK; +} + +static ngx_int_t ngx_http_brotli_ratio_variable(ngx_http_request_t* r, + ngx_http_variable_value_t* v, + uintptr_t data) { + ngx_uint_t ratio_int; + ngx_uint_t ratio_frac; + ngx_http_brotli_ctx_t* ctx; + + v->valid = 1; + v->no_cacheable = 0; + v->not_found = 0; + + ctx = ngx_http_get_module_ctx(r, ngx_http_brotli_filter_module); + + /* Only report variable on non-failing streams. */ + if (ctx == NULL || !ctx->success) { + v->not_found = 1; + return NGX_OK; + } + + v->data = ngx_pnalloc(r->pool, NGX_INT32_LEN + 3); + if (v->data == NULL) { + return NGX_ERROR; + } + + ratio_int = (ngx_uint_t)(ctx->bytes_in / ctx->bytes_out); + ratio_frac = (ngx_uint_t)((ctx->bytes_in * 100 / ctx->bytes_out) % 100); + + /* Rounding; e.g. 2.125 to 2.13 */ + if ((ctx->bytes_in * 1000 / ctx->bytes_out) % 10 > 4) { + ratio_frac++; + if (ratio_frac > 99) { + ratio_int++; + ratio_frac = 0; + } + } + + v->len = ngx_sprintf(v->data, "%ui.%02ui", ratio_int, ratio_frac) - v->data; + + return NGX_OK; +} + +static void* ngx_http_brotli_create_conf(ngx_conf_t* cf) { + ngx_http_brotli_conf_t* conf; + + conf = ngx_pcalloc(cf->pool, sizeof(ngx_http_brotli_conf_t)); + if (conf == NULL) { + return NULL; + } + + /* ngx_pcalloc fills result with zeros -> + conf->bufs.num = 0; + conf->types = { NULL }; + conf->types_keys = NULL; */ + + conf->enable = NGX_CONF_UNSET; + + conf->quality = NGX_CONF_UNSET; + conf->lg_win = NGX_CONF_UNSET_SIZE; + conf->min_length = NGX_CONF_UNSET; + + return conf; +} + +static char* ngx_http_brotli_merge_conf(ngx_conf_t* cf, void* parent, + void* child) { + ngx_http_brotli_conf_t* prev = parent; + ngx_http_brotli_conf_t* conf = child; + char* rc; + + ngx_conf_merge_value(conf->enable, prev->enable, 0); + + ngx_conf_merge_value(conf->quality, prev->quality, 6); + ngx_conf_merge_size_value(conf->lg_win, prev->lg_win, 19); + ngx_conf_merge_value(conf->min_length, prev->min_length, 20); + + rc = ngx_http_merge_types(cf, &conf->types_keys, &conf->types, + &prev->types_keys, &prev->types, + ngx_http_html_default_types); + if (rc != NGX_CONF_OK) { + return NGX_CONF_ERROR; + } + + return NGX_CONF_OK; +} + +/* Prepend to filter chain. */ +static ngx_int_t ngx_http_brotli_filter_init(ngx_conf_t* cf) { + ngx_http_next_header_filter = ngx_http_top_header_filter; + ngx_http_top_header_filter = ngx_http_brotli_header_filter; + + ngx_http_next_body_filter = ngx_http_top_body_filter; + ngx_http_top_body_filter = ngx_http_brotli_body_filter; + + return NGX_OK; +} + +/* Translate "window size" to window bits (log2), and check bounds. */ +static char* ngx_http_brotli_parse_wbits(ngx_conf_t* cf, void* post, + void* data) { + size_t* parameter = data; + size_t bits; + size_t wsize; + + for (bits = BROTLI_MIN_WINDOW_BITS; bits <= BROTLI_MAX_WINDOW_BITS; bits++) { + wsize = 1u << bits; + if (*parameter == wsize) { + *parameter = bits; + return NGX_CONF_OK; + } + } + + return "must be 1k, 2k, 4k, 8k, 16k, 32k, 64k, 128k, 256k, 512k, 1m, 2m, 4m, " + "8m or 16m"; +} diff --git a/script/.travis-before-test.sh b/script/.travis-before-test.sh new file mode 100755 index 000000000..9f6916449 --- /dev/null +++ b/script/.travis-before-test.sh @@ -0,0 +1,24 @@ +#!/bin/bash +set -ex + +# Setup shortcuts. +ROOT=`pwd` +FILES=$ROOT/script/test + +# Setup directory structure. +cd $ROOT/script +if [ ! -d test ]; then + mkdir test +fi +cd test +if [ ! -d logs ]; then + mkdir logs +fi + +# Download sample texts. +curl --compressed -o $FILES/war-and-peace.txt http://www.gutenberg.org/files/2600/2600-0.txt +echo "Kot lomom kolol slona!" > $FILES/small.txt +echo "Kot lomom kolol slona!" > $FILES/small.html + +# Restore status-quo. +cd $ROOT diff --git a/script/.travis-compile.sh b/script/.travis-compile.sh new file mode 100755 index 000000000..2fa01d616 --- /dev/null +++ b/script/.travis-compile.sh @@ -0,0 +1,29 @@ +#!/bin/bash +set -ex + +# Setup shortcuts. +ROOT=`pwd` + +# Clone nginx read-only git repository. +if [ ! -d "nginx" ]; then + git clone https://github.com/nginx/nginx.git +fi + +# Build nginx + filter module. +cd $ROOT/nginx +# Pro memoria: --with-debug +./auto/configure \ + --prefix=$ROOT/script/test \ + --with-http_v2_module \ + --add-module=$ROOT +make -j 16 + +# Build brotli CLI. +cd $ROOT/deps/brotli +mkdir out +cd out +cmake .. +make -j 16 brotli + +# Restore status-quo. +cd $ROOT diff --git a/script/.travis-test.sh b/script/.travis-test.sh new file mode 100755 index 000000000..1db8d157b --- /dev/null +++ b/script/.travis-test.sh @@ -0,0 +1,168 @@ +#!/bin/bash + +# Setup shortcuts. +ROOT=`pwd` +NGINX=$ROOT/nginx/objs/nginx +BROTLI=$ROOT/deps/brotli/out/brotli +SERVER=http://localhost:8080 +FILES=$ROOT/script/test +HR="---------------------------------------------------------------------------" + +if [ ! -d tmp ]; then + mkdir tmp +fi + +rm tmp/* + +add_result() { + echo $1 >&2 + echo $1 >> tmp/results.log +} + +get_failed() { + echo `cat tmp/results.log | grep -v OK | wc -l` +} + +get_count() { + echo `cat tmp/results.log | wc -l` +} + +expect_equal() { + expected=$1 + actual=$2 + if cmp $expected $actual; then + add_result "OK" + else + add_result "FAIL (equality)" + fi +} + +expect_br_equal() { + expected=$1 + actual_br=$2 + if $BROTLI -dfk ./${actual_br}.br; then + expect_equal $expected $actual_br + else + add_result "FAIL (decompression)" + fi +} + +################################################################################ + +# Start default server. +echo "Statring NGINX" +$NGINX -c $ROOT/script/test.conf +# Fetch vanilla 404 response. +curl -s -o tmp/notfound.txt $SERVER/notfound + +CURL="curl -s" + +# Run tests. +echo $HR + +echo "Test: long file with rate limit" +$CURL -H 'Accept-encoding: br' -o tmp/war-and-peace.br --limit-rate 300K $SERVER/war-and-peace.txt +expect_br_equal $FILES/war-and-peace.txt tmp/war-and-peace + +echo "Test: compressed 404" +$CURL -H 'Accept-encoding: br' -o tmp/notfound.br $SERVER/notfound +expect_br_equal tmp/notfound.txt tmp/notfound + +echo "Test: A-E: 'gzip, br'" +$CURL -H 'Accept-encoding: gzip, br' -o tmp/ae-01.br $SERVER/small.txt +expect_br_equal $FILES/small.txt tmp/ae-01 + +echo "Test: A-E: 'gzip, br, deflate'" +$CURL -H 'Accept-encoding: gzip, br, deflate' -o tmp/ae-02.br $SERVER/small.txt +expect_br_equal $FILES/small.txt tmp/ae-02 + +echo "Test: A-E: 'gzip, br;q=1, deflate'" +$CURL -H 'Accept-encoding: gzip, br;q=1, deflate' -o tmp/ae-03.br $SERVER/small.txt +expect_br_equal $FILES/small.txt tmp/ae-03 + +echo "Test: A-E: 'br;q=0.001'" +$CURL -H 'Accept-encoding: br;q=0.001' -o tmp/ae-04.br $SERVER/small.txt +expect_br_equal $FILES/small.txt tmp/ae-04 + +echo "Test: A-E: 'bro'" +$CURL -H 'Accept-encoding: bro' -o tmp/ae-05.txt $SERVER/small.txt +expect_equal $FILES/small.txt tmp/ae-05.txt + +echo "Test: A-E: 'bo'" +$CURL -H 'Accept-encoding: bo' -o tmp/ae-06.txt $SERVER/small.txt +expect_equal $FILES/small.txt tmp/ae-06.txt + +echo "Test: A-E: 'br;q=0'" +$CURL -H 'Accept-encoding: br;q=0' -o tmp/ae-07.txt $SERVER/small.txt +expect_equal $FILES/small.txt tmp/ae-07.txt + +echo "Test: A-E: 'br;q=0.'" +$CURL -H 'Accept-encoding: br;q=0.' -o tmp/ae-08.txt $SERVER/small.txt +expect_equal $FILES/small.txt tmp/ae-08.txt + +echo "Test: A-E: 'br;q=0.0'" +$CURL -H 'Accept-encoding: br;q=0.0' -o tmp/ae-09.txt $SERVER/small.txt +expect_equal $FILES/small.txt tmp/ae-09.txt + +echo "Test: A-E: 'br;q=0.00'" +$CURL -H 'Accept-encoding: br;q=0.00' -o tmp/ae-10.txt $SERVER/small.txt +expect_equal $FILES/small.txt tmp/ae-10.txt + +echo "Test: A-E: 'br ; q = 0.000'" +$CURL -H 'Accept-encoding: br ; q = 0.000' -o tmp/ae-11.txt $SERVER/small.txt +expect_equal $FILES/small.txt tmp/ae-11.txt + +echo "Test: A-E: 'bar'" +$CURL -H 'Accept-encoding: bar' -o tmp/ae-12.txt $SERVER/small.html +expect_equal $FILES/small.html tmp/ae-12.txt + +echo "Test: A-E: 'b'" +$CURL -H 'Accept-encoding: b' -o tmp/ae-13.txt $SERVER/small.html +expect_equal $FILES/small.html tmp/ae-13.txt + +echo $HR +echo "Stopping default NGINX" +# Stop server. +$NGINX -c $ROOT/script/test.conf -s stop + +################################################################################ + +# Start default server. +echo "Statring h2 NGINX" +$NGINX -c $ROOT/script/test_h2.conf + +CURL="curl --http2-prior-knowledge -s" + +# Run tests. +echo $HR + +echo "Test: long file with rate limit" +$CURL -H 'Accept-encoding: br' -o tmp/h2-war-and-peace.br --limit-rate 300K $SERVER/war-and-peace.txt +expect_br_equal $FILES/war-and-peace.txt tmp/h2-war-and-peace + +echo "Test: A-E: 'gzip, br'" +$CURL -H 'Accept-encoding: gzip, br' -o tmp/h2-ae-01.br $SERVER/small.txt +expect_br_equal $FILES/small.txt tmp/h2-ae-01 + +echo "Test: A-E: 'b'" +$CURL -H 'Accept-encoding: b' -o tmp/h2-ae-13.txt $SERVER/small.html +expect_equal $FILES/small.html tmp/h2-ae-13.txt + +echo $HR +echo "Stopping h2 NGINX" +# Stop server. +$NGINX -c $ROOT/script/test_h2.conf -s stop + +################################################################################ + +# Report. + +FAILED=$(get_failed $STATUS) +COUNT=$(get_count $STATUS) +echo $HR +echo "Results: $FAILED of $COUNT tests failed" + +# Restore status-quo. +cd $ROOT + +exit $FAILED diff --git a/script/test.conf b/script/test.conf new file mode 100644 index 000000000..93957f960 --- /dev/null +++ b/script/test.conf @@ -0,0 +1,32 @@ +events { + worker_connections 4; +} + +daemon on; +error_log /dev/stdout info; + +http { + access_log ./access.log; + error_log ./error.log; + + gzip on; + gzip_comp_level 1; + gzip_types text/plain text/css; + + brotli on; + brotli_comp_level 1; + brotli_types text/plain text/css; + + server { + listen 8080 default_server; + listen [::]:8080 default_server; + + root ./; + + index index.html; + + location / { + try_files $uri $uri/ =404; + } + } +} diff --git a/script/test_h2.conf b/script/test_h2.conf new file mode 100644 index 000000000..0f2c791b4 --- /dev/null +++ b/script/test_h2.conf @@ -0,0 +1,32 @@ +events { + worker_connections 4; +} + +daemon on; +error_log /dev/stdout info; + +http { + access_log ./access.log; + error_log ./error.log; + + gzip on; + gzip_comp_level 1; + gzip_types text/plain text/css; + + brotli on; + brotli_comp_level 1; + brotli_types text/plain text/css; + + server { + listen 8080 http2; + listen [::]:8080 http2; + + root ./; + + index index.html; + + location / { + try_files $uri $uri/ =404; + } + } +} diff --git a/static/config b/static/config new file mode 100644 index 000000000..de07d1ce2 --- /dev/null +++ b/static/config @@ -0,0 +1,54 @@ +# Copyright (C) 2015-2019 Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. + +if [ "$ngx_addon_name" = "ngx_brotli" ]; then + BROTLI_MODULE_SRC_DIR="$ngx_addon_dir/static" +else + BROTLI_MODULE_SRC_DIR="$ngx_addon_dir" +fi + +ngx_addon_name=ngx_brotli_static + +if [ -z "$ngx_module_link" ]; then +cat << END + +$0: error: Brotli module requires recent version of NGINX (1.9.11+). + +END + exit 1 +fi + +ngx_module_type=HTTP +ngx_module_name=ngx_http_brotli_static_module +ngx_module_incs= +ngx_module_deps= +ngx_module_srcs="$BROTLI_MODULE_SRC_DIR/ngx_http_brotli_static_module.c" +ngx_module_libs= +ngx_module_order= + +. auto/module + +have=NGX_HTTP_GZIP . auto/have +have=NGX_HTTP_BROTLI_STATIC . auto/have +have=NGX_HTTP_BROTLI_STATIC_MODULE . auto/have # deprecated diff --git a/static/ngx_http_brotli_static_module.c b/static/ngx_http_brotli_static_module.c new file mode 100644 index 000000000..8f9617734 --- /dev/null +++ b/static/ngx_http_brotli_static_module.c @@ -0,0 +1,320 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) Nginx, Inc. + * Copyright (C) Google Inc. + */ + +#include +#include +#include + +/* >> Configuration */ + +#define NGX_HTTP_BROTLI_STATIC_OFF 0 +#define NGX_HTTP_BROTLI_STATIC_ON 1 +#define NGX_HTTP_BROTLI_STATIC_ALWAYS 2 + +typedef struct { + ngx_uint_t enable; +} configuration_t; + +static ngx_conf_enum_t kBrotliStaticEnum[] = { + {ngx_string("off"), NGX_HTTP_BROTLI_STATIC_OFF}, + {ngx_string("on"), NGX_HTTP_BROTLI_STATIC_ON}, + {ngx_string("always"), NGX_HTTP_BROTLI_STATIC_ALWAYS}, + {ngx_null_string, 0}}; + +/* << Configuration */ + +/* >> Forward declarations */ + +static ngx_int_t handler(ngx_http_request_t* req); +static void* create_conf(ngx_conf_t* root_cfg); +static char* merge_conf(ngx_conf_t* root_cfg, void* parent, void* child); +static ngx_int_t init(ngx_conf_t* root_cfg); + +/* << Forward declarations*/ + +/* >> Module definition */ + +static ngx_command_t kCommands[] = { + {ngx_string("brotli_static"), + NGX_HTTP_MAIN_CONF | NGX_HTTP_SRV_CONF | NGX_HTTP_LOC_CONF | + NGX_CONF_TAKE1, + ngx_conf_set_enum_slot, NGX_HTTP_LOC_CONF_OFFSET, + offsetof(configuration_t, enable), &kBrotliStaticEnum}, + ngx_null_command}; + +static ngx_http_module_t kModuleContext = { + NULL, /* preconfiguration */ + init, /* postconfiguration */ + + NULL, /* create main configuration */ + NULL, /* init main configuration */ + + NULL, /* create server configuration */ + NULL, /* merge server configuration */ + + create_conf, /* create location configuration */ + merge_conf /* merge location configuration */ +}; + +ngx_module_t ngx_http_brotli_static_module = { + NGX_MODULE_V1, + &kModuleContext, /* module context */ + kCommands, /* module directives */ + NGX_HTTP_MODULE, /* module type */ + NULL, /* init master */ + NULL, /* init module */ + NULL, /* init process */ + NULL, /* init thread */ + NULL, /* exit thread */ + NULL, /* exit process */ + NULL, /* exit master */ + NGX_MODULE_V1_PADDING}; + +/* << Module definition*/ + +static const u_char kContentEncoding[] = "Content-Encoding"; +static /* const */ char kEncoding[] = "br"; +static const size_t kEncodingLen = 2; +static /* const */ u_char kSuffix[] = ".br"; +static const size_t kSuffixLen = 3; + +static ngx_int_t check_accept_encoding(ngx_http_request_t* req) { + ngx_table_elt_t* accept_encoding_entry; + ngx_str_t* accept_encoding; + u_char* cursor; + u_char* end; + u_char before; + u_char after; + + accept_encoding_entry = req->headers_in.accept_encoding; + if (accept_encoding_entry == NULL) return NGX_DECLINED; + accept_encoding = &accept_encoding_entry->value; + + cursor = accept_encoding->data; + end = cursor + accept_encoding->len; + while (1) { + u_char digit; + /* It would be an idiotic idea to rely on compiler to produce performant + binary, that is why we just do -1 at every call site. */ + cursor = ngx_strcasestrn(cursor, kEncoding, kEncodingLen - 1); + if (cursor == NULL) return NGX_DECLINED; + before = (cursor == accept_encoding->data) ? ' ' : cursor[-1]; + cursor += kEncodingLen; + after = (cursor >= end) ? ' ' : *cursor; + if (before != ',' && before != ' ') continue; + if (after != ',' && after != ' ' && after != ';') continue; + + /* Check for ";q=0[.[0[0[0]]]]" */ + while (*cursor == ' ') cursor++; + if (*(cursor++) != ';') break; + while (*cursor == ' ') cursor++; + if (*(cursor++) != 'q') break; + while (*cursor == ' ') cursor++; + if (*(cursor++) != '=') break; + while (*cursor == ' ') cursor++; + if (*(cursor++) != '0') break; + if (*(cursor++) != '.') return NGX_DECLINED; /* ;q=0, */ + digit = *(cursor++); + if (digit < '0' || digit > '9') return NGX_DECLINED; /* ;q=0., */ + if (digit > '0') break; + digit = *(cursor++); + if (digit < '0' || digit > '9') return NGX_DECLINED; /* ;q=0.0, */ + if (digit > '0') break; + digit = *(cursor++); + if (digit < '0' || digit > '9') return NGX_DECLINED; /* ;q=0.00, */ + if (digit > '0') break; + return NGX_DECLINED; /* ;q=0.000 */ + } + return NGX_OK; +} + +/* Test if this request is allowed to have the brotli response. */ +static ngx_int_t check_eligility(ngx_http_request_t* req) { + if (req != req->main) return NGX_DECLINED; + if (check_accept_encoding(req) != NGX_OK) return NGX_DECLINED; + req->gzip_tested = 1; + req->gzip_ok = 0; + return NGX_OK; +} + +static ngx_int_t handler(ngx_http_request_t* req) { + configuration_t* cfg; + ngx_int_t rc; + u_char* last; + ngx_str_t path; + size_t root; + ngx_log_t* log; + ngx_http_core_loc_conf_t* location_cfg; + ngx_open_file_info_t file_info; + ngx_table_elt_t* content_encoding_entry; + ngx_buf_t* buf; + ngx_chain_t out; + + /* Only GET and HEAD requensts are supported. */ + if (!(req->method & (NGX_HTTP_GET | NGX_HTTP_HEAD))) return NGX_DECLINED; + + /* Only files are supported. */ + if (req->uri.data[req->uri.len - 1] == '/') return NGX_DECLINED; + + /* Get configuration and check if module is disabled. */ + cfg = ngx_http_get_module_loc_conf(req, ngx_http_brotli_static_module); + if (cfg->enable == NGX_HTTP_BROTLI_STATIC_OFF) return NGX_DECLINED; + + if (cfg->enable == NGX_HTTP_BROTLI_STATIC_ALWAYS) { + /* Ignore request properties (e.g. Accept-Encoding). */ + } else { + /* NGX_HTTP_BROTLI_STATIC_ON */ + req->gzip_vary = 1; + rc = check_eligility(req); + if (rc != NGX_OK) return NGX_DECLINED; + } + + /* Get path and append the suffix. */ + last = ngx_http_map_uri_to_path(req, &path, &root, kSuffixLen); + if (last == NULL) return NGX_HTTP_INTERNAL_SERVER_ERROR; + /* +1 for reinstating the terminating 0. */ + ngx_cpystrn(last, kSuffix, kSuffixLen + 1); + path.len += kSuffixLen; + + log = req->connection->log; + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, log, 0, "http filename: \"%s\"", + path.data); + + /* Prepare to read the file. */ + location_cfg = ngx_http_get_module_loc_conf(req, ngx_http_core_module); + ngx_memzero(&file_info, sizeof(ngx_open_file_info_t)); + file_info.read_ahead = location_cfg->read_ahead; + file_info.directio = location_cfg->directio; + file_info.valid = location_cfg->open_file_cache_valid; + file_info.min_uses = location_cfg->open_file_cache_min_uses; + file_info.errors = location_cfg->open_file_cache_errors; + file_info.events = location_cfg->open_file_cache_events; + rc = ngx_http_set_disable_symlinks(req, location_cfg, &path, &file_info); + if (rc != NGX_OK) return NGX_HTTP_INTERNAL_SERVER_ERROR; + + /* Try to fetch file and process errors. */ + rc = ngx_open_cached_file(location_cfg->open_file_cache, &path, &file_info, + req->pool); + if (rc != NGX_OK) { + ngx_uint_t level; + switch (file_info.err) { + case 0: + return NGX_HTTP_INTERNAL_SERVER_ERROR; + + case NGX_ENOENT: + case NGX_ENOTDIR: + case NGX_ENAMETOOLONG: + return NGX_DECLINED; + +#if (NGX_HAVE_OPENAT) + case NGX_EMLINK: + case NGX_ELOOP: +#endif + case NGX_EACCES: + level = NGX_LOG_ERR; + break; + + default: + level = NGX_LOG_CRIT; + break; + } + ngx_log_error(level, log, file_info.err, "%s \"%s\" failed", + file_info.failed, path.data); + return NGX_DECLINED; + } + + /* So far so good. */ + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, log, 0, "http static fd: %d", + file_info.fd); + + /* Only files are supported. */ + if (file_info.is_dir) { + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, log, 0, "http dir"); + return NGX_DECLINED; + } +#if !(NGX_WIN32) + if (!file_info.is_file) { + ngx_log_error(NGX_LOG_CRIT, log, 0, "\"%s\" is not a regular file", + path.data); + return NGX_HTTP_NOT_FOUND; + } +#endif + + /* Prepare request push the body. */ + req->root_tested = !req->error_page; + rc = ngx_http_discard_request_body(req); + if (rc != NGX_OK) return rc; + log->action = "sending response to client"; + req->headers_out.status = NGX_HTTP_OK; + req->headers_out.content_length_n = file_info.size; + req->headers_out.last_modified_time = file_info.mtime; + rc = ngx_http_set_etag(req); + if (rc != NGX_OK) return NGX_HTTP_INTERNAL_SERVER_ERROR; + rc = ngx_http_set_content_type(req); + if (rc != NGX_OK) return NGX_HTTP_INTERNAL_SERVER_ERROR; + + /* Set "Content-Encoding" header. */ + content_encoding_entry = ngx_list_push(&req->headers_out.headers); + if (content_encoding_entry == NULL) return NGX_HTTP_INTERNAL_SERVER_ERROR; + content_encoding_entry->hash = 1; + ngx_str_set(&content_encoding_entry->key, kContentEncoding); + ngx_str_set(&content_encoding_entry->value, kEncoding); + req->headers_out.content_encoding = content_encoding_entry; + + /* Setup response body. */ + buf = ngx_pcalloc(req->pool, sizeof(ngx_buf_t)); + if (buf == NULL) return NGX_HTTP_INTERNAL_SERVER_ERROR; + buf->file = ngx_pcalloc(req->pool, sizeof(ngx_file_t)); + if (buf->file == NULL) return NGX_HTTP_INTERNAL_SERVER_ERROR; + buf->file_pos = 0; + buf->file_last = file_info.size; + buf->in_file = buf->file_last ? 1 : 0; + buf->last_buf = (req == req->main) ? 1 : 0; + buf->last_in_chain = 1; + buf->file->fd = file_info.fd; + buf->file->name = path; + buf->file->log = log; + buf->file->directio = file_info.is_directio; + out.buf = buf; + out.next = NULL; + + /* Push the response header. */ + rc = ngx_http_send_header(req); + if (rc == NGX_ERROR || rc > NGX_OK || req->header_only) { + return rc; + } + + /* Push the response body. */ + return ngx_http_output_filter(req, &out); +} + +static void* create_conf(ngx_conf_t* root_cfg) { + configuration_t* cfg; + cfg = ngx_palloc(root_cfg->pool, sizeof(configuration_t)); + if (cfg == NULL) return NULL; + cfg->enable = NGX_CONF_UNSET_UINT; + return cfg; +} + +static char* merge_conf(ngx_conf_t* root_cfg, void* parent, void* child) { + configuration_t* prev = parent; + configuration_t* cfg = child; + ngx_conf_merge_uint_value(cfg->enable, prev->enable, + NGX_HTTP_BROTLI_STATIC_OFF); + return NGX_CONF_OK; +} + +static ngx_int_t init(ngx_conf_t* root_cfg) { + ngx_http_core_main_conf_t* core_cfg; + ngx_http_handler_pt* handler_slot; + core_cfg = ngx_http_conf_get_module_main_conf(root_cfg, ngx_http_core_module); + handler_slot = + ngx_array_push(&core_cfg->phases[NGX_HTTP_CONTENT_PHASE].handlers); + if (handler_slot == NULL) return NGX_ERROR; + *handler_slot = handler; + return NGX_OK; +}