diff --git a/.github/workflows/erlang.yml b/.github/workflows/erlang.yml new file mode 100644 index 00000000..c11fdb57 --- /dev/null +++ b/.github/workflows/erlang.yml @@ -0,0 +1,35 @@ +name: Erlang CI + +on: + push: + branches: [ develop ] + pull_request: + branches: [ develop ] + + +jobs: + + build: + + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + otp: + - "25.1" + - "24.3" + - "22.3" + + container: + image: erlang:${{ matrix.otp }} + + steps: + - uses: lukka/get-cmake@latest + - uses: actions/checkout@v2 + - name: Compile + run: ./rebar3 compile + - name: Run xref and dialyzer + run: ./rebar3 do xref, dialyzer + - name: Run eunit + run: ./rebar3 as gha do eunit diff --git a/.gitignore b/.gitignore index d7b690f2..34d39c51 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ doc/* /.rebar _build .DS_Store +rebar.lock diff --git a/.travis.yml b/.travis.yml index e16bd097..3df2a610 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,10 @@ language: erlang sudo: false install: "true" # don't let travis run get-deps otp_release: + - 21.0 + - 20.0 + - 19.1 + - 18.3 - 17.5 - R16B03-1 ## These won't work until we have checks for application:ensure_all_started diff --git a/README.md b/README.md index 8c455224..131789fe 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ contact [@seancribbs](http://github.com/seancribbs) to get involved. ### Overview -[![Build Status](https://travis-ci.org/webmachine/webmachine.svg?branch=develop)](https://travis-ci.org/webmachine/webmachine) +[![Erlang CI Actions Status](https://github.com/basho/webmachine/workflows/Erlang%20CI/badge.svg)](https://github.com/basho/webmachine/actions) Webmachine is an application layer that adds HTTP semantic awareness on top of the excellent bit-pushing and HTTP syntax-management @@ -46,7 +46,7 @@ and easily create a new `webmachine` application. ``` $ mkdir -p ~/.config/rebar3/templates -$ git clone git://github.com/webmachine/webmachine-rebar3-template.git ~/.config/rebar3/templates +$ git clone https://github.com/webmachine/webmachine-rebar3-template.git ~/.config/rebar3/templates $ rebar3 new webmachine your_app_here ``` diff --git a/include/webmachine.hrl b/include/webmachine.hrl index 5043ad8a..6cebaeb4 100644 --- a/include/webmachine.hrl +++ b/include/webmachine.hrl @@ -1,6 +1 @@ --export([ping/2]). - -include("wm_reqdata.hrl"). - -ping(ReqData, State) -> - {pong, ReqData, State}. diff --git a/include/webmachine_logger.hrl b/include/webmachine_logger.hrl index f1c1af6e..82a15a13 100644 --- a/include/webmachine_logger.hrl +++ b/include/webmachine_logger.hrl @@ -9,8 +9,8 @@ version, response_code, response_length, - end_time :: tuple(), - finish_time :: tuple(), + end_time :: undefined | tuple(), + finish_time :: undefined | tuple(), notes}). -type wm_log_data() :: #wm_log_data{}. diff --git a/include/wm_compat.hrl b/include/wm_compat.hrl new file mode 100644 index 00000000..672f2619 --- /dev/null +++ b/include/wm_compat.hrl @@ -0,0 +1,7 @@ +-ifndef(deprecate_stacktrace). +-define(STPATTERN(Pattern), Pattern). +-define(STACKTRACE, erlang:get_stacktrace()). +-else. +-define(STPATTERN(Pattern), Pattern:__STACKTRACE). +-define(STACKTRACE, __STACKTRACE). +-endif. diff --git a/include/wm_reqdata.hrl b/include/wm_reqdata.hrl index fc0aa2e8..b6841599 100644 --- a/include/wm_reqdata.hrl +++ b/include/wm_reqdata.hrl @@ -1,8 +1,47 @@ --record(wm_reqdata, {method, scheme, version, peer, sock, wm_state, - disp_path, path, raw_path, path_info, path_tokens, - app_root,response_code,max_recv_body, max_recv_hunk, - req_cookie, req_qs, req_headers, req_body, - resp_redirect, resp_headers, resp_body, resp_range, - host_tokens, port, notes - }). +%% Reverse Engineering of these types. Right now based on what +%% mochiweb expected, and the assumptions it made from OTP +%% The following types are clear in erlang:decode_packet/3 +%% https://github.com/erlang/otp/blob/86d1fb0865193cce4e308baa6472885a81033f10/erts/preloaded/src/erlang.erl#L537-L632 +%% {http_request, Method, Url, Version} +%% Method :: atom() | string() +%% Url :: string() +%% Version :: {integer(), integer()} + +-record( + wm_reqdata, + { + method :: wrq:method(), + scheme :: wrq:scheme(), + version :: {non_neg_integer(), % decode_packet/3 + non_neg_integer()}, + peer="defined_in_wm_req_srv_init", + sock="defined_in_wm_req_srv_init", + wm_state = defined_on_call, + disp_path=defined_in_load_dispatch_data, + path = "defined_in_create" :: string(), + raw_path :: string(), % mochiweb:uri/1 + path_info = orddict:new() :: orddict:orddict(), + path_tokens=defined_in_load_dispatch_data, + app_root="defined_in_load_dispatch_data", + response_code = 500 :: non_neg_integer() + | {non_neg_integer(), string()}, + max_recv_body = (1024*(1024*1024)) :: non_neg_integer(), + % Stolen from R13B03 inet_drv.c's TCP_MAX_PACKET_SIZE definition + max_recv_hunk = (64*(1024*1024)) :: non_neg_integer(), + req_cookie = defined_in_create :: [{string(), string()}] + | defined_in_create, + req_qs = defined_in_create :: [{string(), string()}] + | defined_in_create, + req_headers :: webmachine:headers(), + req_body=not_fetched_yet, + resp_redirect = false :: boolean(), + resp_headers = webmachine_headers:empty() :: webmachine:headers(), + resp_body = <<>> :: webmachine:response_body(), + %% follow_request : range responce for range request, normal responce for non-range one + %% ignore_request : normal resopnse for either range reuqest or non-range one + resp_range = follow_request :: follow_request | ignore_request, + host_tokens, + port :: undefined | inet:port_number(), + notes = [] :: list() + }). diff --git a/include/wm_reqstate.hrl b/include/wm_reqstate.hrl index d3d719de..d788f816 100644 --- a/include/wm_reqstate.hrl +++ b/include/wm_reqstate.hrl @@ -1,11 +1,11 @@ --record(wm_reqstate, {socket=undefined, - metadata=orddict:new(), - range=undefined, - peer=undefined, - sock=undefined, - reqdata=undefined, - bodyfetch=undefined, - reqbody=undefined, - log_data=undefined - }). - +-record(wm_reqstate, { + socket=undefined, + metadata=orddict:new(), + range=undefined, + peer=undefined, + sock=undefined, + reqdata=undefined, + bodyfetch=undefined, + reqbody=undefined, + log_data=undefined + }). diff --git a/include/wm_resource.hrl b/include/wm_resource.hrl index 588405df..3381669c 100644 --- a/include/wm_resource.hrl +++ b/include/wm_resource.hrl @@ -1 +1 @@ --record(wm_resource, {module, modstate, modexports, trace}). +-record(wm_resource, {module, modstate, trace}). diff --git a/rebar.config b/rebar.config index 099dab50..cb074eb2 100644 --- a/rebar.config +++ b/rebar.config @@ -1,11 +1,12 @@ %%-*- mode: erlang -*- +{minimum_otp_vsn, "22.0"}. {erl_opts, [warnings_as_errors]}. {cover_enabled, true}. {edoc_opts, [{preprocess, true}]}. {xref_checks, [undefined_function_calls]}. -{deps, [mochiweb]}. +{deps, [{mochiweb, {git, "https://github.com/basho/mochiweb.git", {branch, "develop"}}}]}. {eunit_opts, [ no_tty, @@ -13,10 +14,10 @@ ]}. {profiles, - [{test, + [{gha, [{erl_opts, [{d, 'GITHUBEXCLUDE'}]}]}, + {test, [{deps, [meck, - {ibrowse, {git, "git://github.com/cmullaparthi/ibrowse.git", {tag, "v4.0.2"}}}, - {eunit_formatters, {git, "git://github.com/seancribbs/eunit_formatters", {branch, "master"}}} + {ibrowse, "4.4.0"} ]}, {erl_opts, [debug_info]} ]} diff --git a/rebar.config.script b/rebar.config.script index 52eee007..9679562d 100644 --- a/rebar.config.script +++ b/rebar.config.script @@ -1,7 +1,7 @@ %% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*- %% ex: ft=erlang ts=4 sw=4 et OtpVersion = erlang:system_info(otp_release), -Config1 = case hd(OtpVersion) =:= $R andalso OtpVersion =< "R15B01" of +Config1 = case hd(OtpVersion) =:= $R andalso OtpVersion < "R15B02" of true -> HashDefine = [{d,old_hash}], case lists:keysearch(erl_opts, 1, CONFIG) of @@ -11,4 +11,15 @@ Config1 = case hd(OtpVersion) =:= $R andalso OtpVersion =< "R15B01" of CONFIG ++ [{erl_opts, HashDefine}] end; false -> CONFIG - end. + end, +if + OtpVersion >= "21" -> + case lists:keysearch(erl_opts, 1, Config1) of + {value, {erl_opts, Opts2}} -> + lists:keyreplace(erl_opts, 1, Config1, {erl_opts, [{d, deprecate_stacktrace}|Opts2]}); + false -> + [{erl_opts, [{d, deprecate_stacktrace}]}|Config1] + end; + true -> + Config1 +end. diff --git a/rebar.lock b/rebar.lock deleted file mode 100644 index 62eb8202..00000000 --- a/rebar.lock +++ /dev/null @@ -1 +0,0 @@ -[{<<"mochiweb">>,{pkg,<<"mochiweb">>,<<"2.12.2">>},0}]. diff --git a/rebar3 b/rebar3 index 81c9d927..a83d554a 100755 Binary files a/rebar3 and b/rebar3 differ diff --git a/src/webmachine.app.src b/src/webmachine.app.src index 3f6832c8..363e92d0 100644 --- a/src/webmachine.app.src +++ b/src/webmachine.app.src @@ -10,9 +10,20 @@ crypto, mochiweb]}, {mod, {webmachine_app, []}}, - {env, []}, + {env, + %% env is THE place for expected defaults. Even if it's just + %% comments, it's worthwhile + [ + {log_handlers, []} + ,{error_handler, webmachine_error_handler} + %% error_handler is a module that implements the function render_error/3 + ,{rewrite_module, undefined} + %% module that has rewrite/5 + ,{server_name, undefined} + %% string() for the "Server" response header + ]}, - {contributors,["Sean Cribbs", "Joe DeVivo" "Bryan Fink", + {maintainers,["Sean Cribbs", "Joe DeVivo", "Bryan Fink", "Kelly McLaughlin", "Jared Morrow", "Andy Gross", "Steve Vinoski"]}, {licenses,["Apache"]}, diff --git a/src/webmachine.erl b/src/webmachine.erl index 902547c7..cad3209d 100644 --- a/src/webmachine.erl +++ b/src/webmachine.erl @@ -17,12 +17,18 @@ -module(webmachine). -author('Justin Sheehy '). -author('Andy Gross '). --export([start/0, stop/0]). --export([new_request/2]). +-export([start/0, stop/0, new_request/2]). --include("webmachine_logger.hrl"). --include("wm_reqstate.hrl"). --include("wm_reqdata.hrl"). +-type headers() :: webmachine_headers:t(). +-type response_body() :: iodata() + | {stream, StreamBody::any()} + | {known_length_stream, non_neg_integer(), StreamBody::any()} + | {stream, non_neg_integer(), fun()} %% TODO: type for fun() + | {writer, WrtieBody::any()} + | {file, IoDevice::any()}. + + +-export_type([headers/0, response_body/0]). %% @spec start() -> ok %% @doc Start the webmachine server. @@ -46,51 +52,8 @@ ensure_started(App) -> stop() -> application:stop(webmachine). -new_request(mochiweb, Request) -> - Method = Request:get(method), - Scheme = Request:get(scheme), - Version = Request:get(version), - {Headers, RawPath} = case application:get_env(webmachine, rewrite_module) of - {ok, RewriteMod} -> - do_rewrite(RewriteMod, - Method, - Scheme, - Version, - Request:get(headers), - Request:get(raw_path)); - undefined -> - {Request:get(headers), Request:get(raw_path)} - end, - Socket = Request:get(socket), - InitState = #wm_reqstate{socket=Socket, - reqdata=wrq:create(Method,Scheme,Version,RawPath,Headers)}, - - InitReq = {webmachine_request,InitState}, - {Peer, _ReqState} = InitReq:get_peer(), - {Sock, ReqState} = InitReq:get_sock(), - ReqData = wrq:set_sock(Sock, - wrq:set_peer(Peer, - ReqState#wm_reqstate.reqdata)), - LogData = #wm_log_data{start_time=os:timestamp(), - method=Method, - headers=Headers, - peer=Peer, - sock=Sock, - path=RawPath, - version=Version, - response_code=404, - response_length=0}, - webmachine_request:new(ReqState#wm_reqstate{log_data=LogData, - reqdata=ReqData}). - -do_rewrite(RewriteMod, Method, Scheme, Version, Headers, RawPath) -> - case RewriteMod:rewrite(Method, Scheme, Version, Headers, RawPath) of - %% only raw path has been rewritten (older style rewriting) - NewPath when is_list(NewPath) -> {Headers, NewPath}; - - %% headers and raw path rewritten (new style rewriting) - {NewHeaders, NewPath} -> {NewHeaders,NewPath} - end. +new_request(mochiweb, MochiReq) -> + webmachine_mochiweb:new_webmachine_req(MochiReq). %% %% TEST diff --git a/src/webmachine_app.erl b/src/webmachine_app.erl index 88116050..0c15a08c 100644 --- a/src/webmachine_app.erl +++ b/src/webmachine_app.erl @@ -27,17 +27,19 @@ -include("webmachine_logger.hrl"). +-define(QUIP, "greased slide to failure"). + %% @spec start(_Type, _StartArgs) -> ServerRet %% @doc application start callback for webmachine. start(_Type, _StartArgs) -> webmachine_deps:ensure(), + + %% Populate dynamic defaults on load: + load_default_app_config(), + {ok, _Pid} = SupLinkRes = webmachine_sup:start_link(), - Handlers = case application:get_env(webmachine, log_handlers) of - undefined -> - []; - {ok, Val} -> - Val - end, + Handlers = application:get_env(webmachine, log_handlers, []), + %% handlers failing to start are handled in the handler_watcher _ = [supervisor:start_child(webmachine_logger_watcher_sup, [?EVENT_LOGGER, Module, Config]) || @@ -48,3 +50,30 @@ start(_Type, _StartArgs) -> %% @doc application stop callback for webmachine. stop(_State) -> ok. + +-spec load_default_app_config() -> ok. +load_default_app_config() -> + + case application:get_env(webmachine, server_name, undefined) of + Name when is_list(Name) -> + ok; + _ -> + set_default_server_name() + end, + ok. + + +set_default_server_name() -> + {mochiweb, _, MochiVersion} = + lists:keyfind(mochiweb, 1, application:loaded_applications()), + + {webmachine, _, WMVersion} = + lists:keyfind(webmachine, 1, application:loaded_applications()), + ServerName = + lists:flatten( + io_lib:format( + "MochiWeb/~s WebMachine/~s (~s)", + [MochiVersion, WMVersion, ?QUIP])), + application:set_env( + webmachine, server_name, ServerName), + ok. diff --git a/src/webmachine_decision_core.erl b/src/webmachine_decision_core.erl index ec157584..34263895 100644 --- a/src/webmachine_decision_core.erl +++ b/src/webmachine_decision_core.erl @@ -22,8 +22,14 @@ -author('Andy Gross '). -author('Bryan Fink '). -export([handle_request/2]). --export([do_log/1]). -include("webmachine_logger.hrl"). +-include("wm_compat.hrl"). + +%% Suppress Erlang/OTP 21 warnings about the new method to retrieve +%% stacktraces. +-ifdef(OTP_RELEASE). +-compile({nowarn_deprecated_function, [{erlang, get_stacktrace, 0}]}). +-endif. handle_request(Resource, ReqState) -> _ = [erase(X) || X <- [decision, code, req_body, bytes_written, tmp_reqstate]], @@ -32,20 +38,20 @@ handle_request(Resource, ReqState) -> try d(v3b13) catch - error:_ -> - error_response(erlang:get_stacktrace()) + ?STPATTERN(error:_Reason) -> + error_response(?STACKTRACE) end. wrcall(X) -> RS0 = get(reqstate), Req = webmachine_request:new(RS0), - {Response, RS1} = Req:call(X), + {Response, RS1} = webmachine_request:call(X, Req), put(reqstate, RS1), Response. resource_call(Fun) -> Resource = get(resource), - {Reply, NewResource, NewRS} = Resource:do(Fun,get()), + {Reply, NewResource, NewRS} = webmachine_resource:do(Fun,get(),Resource), put(resource, NewResource), put(reqstate, NewRS), Reply. @@ -97,7 +103,7 @@ finish_response({Code, _}=CodeAndPhrase, Resource, EndTime) -> end_time=EndTime, notes=Notes}, spawn(fun() -> do_log(LogData) end), - Resource:stop(). + webmachine_resource:stop(Resource). error_response(Reason) -> error_response(500, Reason). @@ -153,12 +159,10 @@ do_log(LogData) -> log_decision(DecisionID) -> Resource = get(resource), - Resource:log_d(DecisionID). + webmachine_resource:log_d(DecisionID, Resource). %% "Service Available" decision(v3b13) -> - decision_test(resource_call(ping), pong, v3b13b, 503); -decision(v3b13b) -> decision_test(resource_call(service_available), true, v3b12, 503); %% "Known method?" decision(v3b12) -> diff --git a/src/webmachine_dispatcher.erl b/src/webmachine_dispatcher.erl index dffa9aa7..280fdbab 100644 --- a/src/webmachine_dispatcher.erl +++ b/src/webmachine_dispatcher.erl @@ -628,7 +628,7 @@ make_reqdata(Path) -> MochiReq = mochiweb_request:new(testing, 'GET', Path, {1, 1}, mochiweb_headers:make([])), Req = webmachine:new_request(mochiweb, MochiReq), - {RD, _} = Req:get_reqdata(), + {RD, _} = webmachine_request:get_reqdata(Req), RD. -endif. diff --git a/src/webmachine_error_handler.erl b/src/webmachine_error_handler.erl index 590e428e..a028b98c 100644 --- a/src/webmachine_error_handler.erl +++ b/src/webmachine_error_handler.erl @@ -26,19 +26,22 @@ -export([render_error/3]). render_error(Code, Req, Reason) -> - case Req:has_response_body() of + case webmachine_request:has_response_body(Req) of {true,_} -> maybe_log(Code, Req, Reason), - Req:response_body(); - {false,_} -> render_error_body(Code, Req:trim_state(), Reason) + webmachine_request:response_body(Req); + {false,_} -> + render_error_body(Code, webmachine_request:trim_state(Req), Reason) end. render_error_body(404, Req, _Reason) -> - {ok, ReqState} = Req:add_response_header("Content-Type", "text/html"), + {ok, ReqState} = + webmachine_request:add_response_header("Content-Type", "text/html", Req), {<<"404 Not Found

Not Found

The requested document was not found on this server.


mochiweb+webmachine web server
">>, ReqState}; render_error_body(500, Req, Reason) -> - {ok, ReqState} = Req:add_response_header("Content-Type", "text/html"), + {ok, ReqState} = + webmachine_request:add_response_header("Content-Type", "text/html", Req), maybe_log(500, Req, Reason), STString = io_lib:format("~p", [Reason]), ErrorStart = "500 Internal Server Error

Internal Server Error

The server encountered an error while processing this request:
",
@@ -47,8 +50,9 @@ render_error_body(500, Req, Reason) ->
     {erlang:iolist_to_binary(ErrorIOList), ReqState};
 
 render_error_body(501, Req, Reason) ->
-    {ok, ReqState} = Req:add_response_header("Content-Type", "text/html"),
-    {Method,_} = Req:method(),
+    {ok, ReqState} =
+        webmachine_request:add_response_header("Content-Type", "text/html", Req),
+    {Method,_} = webmachine_request:method(Req),
     webmachine_log:log_error(501, Req, Reason),
     ErrorStr = io_lib:format("501 Not Implemented"
                              "

Not Implemented

" @@ -59,7 +63,8 @@ render_error_body(501, Req, Reason) -> {erlang:iolist_to_binary(ErrorStr), ReqState}; render_error_body(503, Req, Reason) -> - {ok, ReqState} = Req:add_response_header("Content-Type", "text/html"), + {ok, ReqState} = + webmachine_request:add_response_header("Content-Type", "text/html", Req), webmachine_log:log_error(503, Req, Reason), ErrorStr = "503 Service Unavailable" "

Service Unavailable

" @@ -71,7 +76,8 @@ render_error_body(503, Req, Reason) -> {list_to_binary(ErrorStr), ReqState}; render_error_body(Code, Req, Reason) -> - {ok, ReqState} = Req:add_response_header("Content-Type", "text/html"), + {ok, ReqState} = + webmachine_request:add_response_header("Content-Type", "text/html", Req), ReasonPhrase = httpd_util:reason_phrase(Code), Body = ["", integer_to_list(Code), diff --git a/src/webmachine_headers.erl b/src/webmachine_headers.erl new file mode 100644 index 00000000..677f7923 --- /dev/null +++ b/src/webmachine_headers.erl @@ -0,0 +1,67 @@ +-module(webmachine_headers). + +-type t() :: gb_trees:tree(name(), value()). +-type name() :: 'Cache-Control' % erlang:decode_packet/3 + | 'Connection' + | 'Date' + | 'Pragma' + | 'Transfer-Encoding' + | 'Upgrade' + | 'Via' + | 'Accept' + | 'Accept-Charset' + | 'Accept-Encoding' + | 'Accept-Language' + | 'Authorization' + | 'From' + | 'Host' + | 'If-Modified-Since' + | 'If-Match' + | 'If-None-Match' + | 'If-Range' + | 'If-Unmodified-Since' + | 'Max-Forwards' + | 'Proxy-Authorization' + | 'Range' + | 'Referer' + | 'User-Agent' + | 'Age' + | 'Location' + | 'Proxy-Authenticate' + | 'Public' + | 'Retry-After' + | 'Server' + | 'Vary' + | 'Warning' + |'Www-Authenticate' + | 'Allow' + | 'Content-Base' + | 'Content-Encoding' + | 'Content-Language' + | 'Content-Length' + | 'Content-Location' + | 'Content-Md5' + | 'Content-Range' + | 'Content-Type' + | 'Etag' + | 'Expires' + | 'Last-Modified' + | 'Accept-Ranges' + | 'Set-Cookie' + | 'Set-Cookie2' + | 'X-Forwarded-For' + | 'Cookie' + | 'Keep-Alive' + | 'Proxy-Connection' + | string() | binary(). + +-type value() :: string() | binary(). +-export_type([t/0, name/0, value/0]). + +-export([ + empty/0 + ]). + +-spec empty() -> t(). +empty() -> + gb_trees:empty(). diff --git a/src/webmachine_log.erl b/src/webmachine_log.erl index 318340e7..724e2d06 100644 --- a/src/webmachine_log.erl +++ b/src/webmachine_log.erl @@ -140,7 +140,7 @@ log_error(LogMsg) -> gen_event:sync_notify(?EVENT_LOGGER, {log_error, LogMsg}). %% @doc Notify registered log event handler of an error event. --spec log_error(pos_integer(), #wm_reqdata{}, term()) -> ok. +-spec log_error(pos_integer(), webmachine_request:t(), term()) -> ok. log_error(Code, Req, Reason) -> gen_event:sync_notify(?EVENT_LOGGER, {log_error, Code, Req, Reason}). diff --git a/src/webmachine_mochiweb.erl b/src/webmachine_mochiweb.erl index 43a864ed..d75ebf9d 100644 --- a/src/webmachine_mochiweb.erl +++ b/src/webmachine_mochiweb.erl @@ -18,7 +18,36 @@ -module(webmachine_mochiweb). -author('Justin Sheehy <justin@basho.com>'). -author('Andy Gross <andy@basho.com>'). --export([start/1, stop/0, stop/1, loop/2]). +-export([start/1, stop/0, stop/1, loop/2, new_webmachine_req/1]). + +-include("webmachine_logger.hrl"). +-include("wm_reqstate.hrl"). +-include("wm_reqdata.hrl"). + +-type mochiweb_request() :: + { + mochiweb_request, + [any()] + }. +%% [any()] always looks like this. Basically it should be a record, +%% but is a list inside a tuple. There are more specific types out +%% there I'm sure, but I have better things to do with my time than +%% typespec a module from 2007. I might come back and make these more +%% specific if I encounter anything worth while, but since they can't +%% be included in the spec anyway, it's worthless(). + +%% [ +%% Socket :: any(), +%% Opts :: any(), +%% Method :: any(), +%% RawPath :: any(), +%% Version :: any(), +%% Headers :: any() +%% ] + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). +-endif. %% The `log_dir' option is deprecated, but remove it from the %% options list if it is present @@ -34,7 +63,7 @@ start(Options) -> webmachine_router:init_routes(DGroup, DispatchList), _ = [application_set_unless_env_or_undef(K, V) || {K, V} <- WMOptions], MochiName = list_to_atom(to_list(PName) ++ "_mochiweb"), - LoopFun = fun(X) -> loop(DGroup, X) end, + LoopFun = {?MODULE, loop, [DGroup]}, mochiweb_http:start([{name, MochiName}, {loop, LoopFun} | OtherOptions]). stop() -> @@ -45,51 +74,120 @@ stop() -> stop(Name) -> mochiweb_http:stop(Name). -loop(Name, MochiReq) -> - Req = webmachine:new_request(mochiweb, MochiReq), - DispatchList = webmachine_router:get_routes(Name), - Host = case host_headers(Req) of - [H|_] -> H; - [] -> [] - end, - {Path, _} = Req:path(), - {RD, _} = Req:get_reqdata(), - - %% Run the dispatch code, catch any errors... - try webmachine_dispatcher:dispatch(Host, Path, DispatchList, RD) of - {no_dispatch_match, _UnmatchedHost, _UnmatchedPathTokens} -> - handle_error(404, {none, none, []}, Req); - {Mod, ModOpts, HostTokens, Port, PathTokens, Bindings, - AppRoot, StringPath} -> - BootstrapResource = webmachine_resource:new(x,x,x,x), - {ok,RS1} = Req:load_dispatch_data(Bindings,HostTokens,Port, - PathTokens,AppRoot,StringPath), - XReq1 = {webmachine_request,RS1}, - try - {ok, Resource} = BootstrapResource:wrap(Mod, ModOpts), - {ok,RS2} = XReq1:set_metadata('resource_module', - resource_module(Mod, ModOpts)), - webmachine_decision_core:handle_request(Resource, RS2) - catch - error:Error -> - handle_error(500, {error, Error}, Req) - end - catch - Type : Error -> - handle_error(500, {Type, Error}, Req) +-spec loop(mochiweb_request(), any()) -> ok. +loop(MochiReq, Name) -> + case new_webmachine_req(MochiReq) of + {{error, NewRequestError}, ErrorReq} -> + handle_error(500, {error, NewRequestError}, ErrorReq); + Req -> + DispatchList = webmachine_router:get_routes(Name), + HostHeaders = host_headers(Req), + Host = host_from_host_values(HostHeaders), + {Path, _} = webmachine_request:path(Req), + {RD, _} = webmachine_request:get_reqdata(Req), + %% Run the dispatch code, catch any errors... + try webmachine_dispatcher:dispatch(Host, Path, DispatchList, RD) of + {no_dispatch_match, _UnmatchedHost, _UnmatchedPathTokens} -> + handle_error(404, {none, none, []}, Req); + {Mod, ModOpts, HostTokens, Port, PathTokens, Bindings, + AppRoot, StringPath} -> + {ok, XReq1} = webmachine_request:load_dispatch_data( + Bindings,HostTokens,Port, + PathTokens,AppRoot,StringPath,Req), + try + {ok, Resource} = webmachine_resource:wrap( + Mod, ModOpts), + {ok, RS2} = webmachine_request:set_metadata( + 'resource_module', + resource_module(Mod, ModOpts), + XReq1), + webmachine_decision_core:handle_request(Resource, RS2) + catch + error:Error -> + handle_error(500, {error, Error}, Req) + end + catch + Type : Error -> + handle_error(500, {Type, Error}, Req) + end + end. + +-spec new_webmachine_req(mochiweb_request()) -> + {module(),#wm_reqstate{}} + |{{error, term()}, #wm_reqstate{}}. +new_webmachine_req(Request) -> + Method = mochiweb_request:get(method, Request), + Scheme = mochiweb_request:get(scheme, Request), + Version = mochiweb_request:get(version, Request), + {Headers, RawPath} = + case application:get_env(webmachine, rewrite_module) of + {ok, undefined} -> + { + mochiweb_request:get(headers, Request), + mochiweb_request:get(raw_path, Request) + }; + {ok, RewriteMod} -> + do_rewrite(RewriteMod, + Method, + Scheme, + Version, + mochiweb_request:get(headers, Request), + mochiweb_request:get(raw_path, Request)) + end, + Socket = mochiweb_request:get(socket, Request), + + InitialReqData = wrq:create(Method,Scheme,Version,RawPath,Headers), + InitialLogData = #wm_log_data{start_time=os:timestamp(), + method=Method, + headers=Headers, + path=RawPath, + version=Version, + response_code=404, + response_length=0}, + + InitState = #wm_reqstate{socket=Socket, + log_data=InitialLogData, + reqdata=InitialReqData}, + InitReq = {webmachine_request,InitState}, + + case webmachine_request:get_peer(InitReq) of + {ErrorGetPeer = {error,_}, ErrorGetPeerReqState} -> + % failed to get peer + { ErrorGetPeer, webmachine_request:new (ErrorGetPeerReqState) }; + {Peer, _ReqState} -> + case webmachine_request:get_sock(InitReq) of + {ErrorGetSock = {error,_}, ErrorGetSockReqState} -> + LogDataWithPeer = InitialLogData#wm_log_data {peer=Peer}, + ReqStateWithSockErr = + ErrorGetSockReqState#wm_reqstate{log_data=LogDataWithPeer}, + { ErrorGetSock, webmachine_request:new (ReqStateWithSockErr) }; + {Sock, ReqState} -> + ReqData = wrq:set_sock(Sock, wrq:set_peer(Peer, InitialReqData)), + LogData = + InitialLogData#wm_log_data {peer=Peer, sock=Sock}, + webmachine_request:new(ReqState#wm_reqstate{log_data=LogData, + reqdata=ReqData}) + end + end. + +do_rewrite(RewriteMod, Method, Scheme, Version, Headers, RawPath) -> + case RewriteMod:rewrite(Method, Scheme, Version, Headers, RawPath) of + %% only raw path has been rewritten (older style rewriting) + NewPath when is_list(NewPath) -> {Headers, NewPath}; + + %% headers and raw path rewritten (new style rewriting) + {NewHeaders, NewPath} -> {NewHeaders,NewPath} end. handle_error(Code, Error, Req) -> {ok, ErrorHandler} = application:get_env(webmachine, error_handler), - {ErrorHTML,ReqState1} = + {ErrorHTML,Req1} = ErrorHandler:render_error(Code, Req, Error), - Req1 = {webmachine_request,ReqState1}, - {ok,ReqState2} = Req1:append_to_response_body(ErrorHTML), - Req2 = {webmachine_request,ReqState2}, - {ok,ReqState3} = Req2:send_response(Code), - Req3 = {webmachine_request,ReqState3}, - {LogData,_ReqState4} = Req3:log_data(), - spawn(webmachine_log, log_access, [LogData]). + {ok,Req2} = webmachine_request:append_to_response_body(ErrorHTML, Req1), + {ok,Req3} = webmachine_request:send_response(Code, Req2), + {LogData,_ReqState4} = webmachine_request:log_data(Req3), + spawn(webmachine_log, log_access, [LogData]), + ok. get_wm_option(OptName, {WMOptions, OtherOptions}) -> {Value, UpdOtherOptions} = @@ -139,8 +237,25 @@ application_set_unless_env(App, Var, Value) -> application:set_env(App, Var, Value) end. +%% X-Forwarded-Host/Server can contain comma-separated values. +%% Reference: https://httpd.apache.org/docs/current/mod/mod_proxy.html#x-headers +%% In that case, we'll take the first as our host, since proxies will append +%% additional values to the original. +host_from_host_values(HostValues) -> + case HostValues of + [] -> + []; + [H|_] -> + case string:tokens(H, ",") of + [FirstHost|_] -> + FirstHost; + [] -> + H + end + end. + host_headers(Req) -> - [ V || {V,_ReqState} <- [Req:get_header_value(H) + [ V || {V,_ReqState} <- [webmachine_request:get_header_value(H, Req) || H <- ["x-forwarded-host", "x-forwarded-server", "host"]], @@ -169,3 +284,33 @@ to_list(L) when is_list(L) -> L; to_list(A) when is_atom(A) -> atom_to_list(A). + +-ifdef(TEST). + +host_from_host_values_test_() -> + [ + {"when a host value is multi-part it resolves the first host correctly", + ?_assertEqual("host1", + host_from_host_values(["host1,host2,host3:443","other", "other1"])) + }, + {"when a host value is multi-part it retains the port", + ?_assertEqual("host1:443", + host_from_host_values(["host1:443,host2","other", "other1"])) + }, + {"a single host per header is resolved correctly", + ?_assertEqual("host1:80", + host_from_host_values(["host1:80","other", "other1"])) + }, + {"a missing host is resolved correctly", + ?_assertEqual([], + host_from_host_values([])) + } + ]. + + %[ + %{"when a host value is multi-part it resolves the first host correctly", + %?_assertEqual("host1:443", + %host_from_host_values(["host1,host2,host3:443","other", "other1"])) } + %]. + +-endif. diff --git a/src/webmachine_perf_log_handler.erl b/src/webmachine_perf_log_handler.erl index 49a5ba9c..ffb50b22 100644 --- a/src/webmachine_perf_log_handler.erl +++ b/src/webmachine_perf_log_handler.erl @@ -132,15 +132,15 @@ fmt_plog(Time, Ip, Method, Path, {VM,Vm}, Status, Length, Mod, TTPD, TTPS) -> non_standard_method_test() -> LogData = #wm_log_data{resource_module=foo, - start_time=now(), + start_time=os:timestamp(), method="FOO", peer={127,0,0,1}, path="/", version={1,1}, response_code=501, response_length=1234, - end_time=now(), - finish_time=now()}, + end_time=os:timestamp(), + finish_time=os:timestamp()}, LogEntry = format_req(LogData), ?assert(is_list(LogEntry)), ok. diff --git a/src/webmachine_request.erl b/src/webmachine_request.erl index 4a5d4783..3f126dc2 100644 --- a/src/webmachine_request.erl +++ b/src/webmachine_request.erl @@ -100,10 +100,11 @@ -include("wm_reqstate.hrl"). -include("wm_reqdata.hrl"). --define(WMVSN, "1.10.9"). --define(QUIP, "cafe not found"). -define(IDLE_TIMEOUT, infinity). +-type t() :: {?MODULE, #wm_reqstate{}}. +-export_type([t/0]). + new(#wm_reqstate{}=ReqState) -> {?MODULE, ReqState}. @@ -147,6 +148,8 @@ get_sock({?MODULE, ReqState} = Req) -> get_sock(ReqState) -> get_sock({?MODULE, ReqState}). +peer_from_peername({error, Error}, _Req) -> + {error, Error}; peer_from_peername({ok, {Addr={10, _, _, _}, _Port}}, Req) -> x_peername(inet_parse:ntoa(Addr), Req); peer_from_peername({ok, {Addr={172, Second, _, _}, _Port}}, Req) @@ -262,7 +265,8 @@ call(do_redirect, {?MODULE, ReqState}) -> reqdata=wrq:do_redirect(true, ReqState#wm_reqstate.reqdata)}}; call({send_response, Code}, Req) when is_integer(Code) -> call({send_response, {Code, undefined}}, Req); -call({send_response, {Code, ReasonPhrase}=CodeAndReason}, Req) when is_integer(Code) -> +call({send_response, {Code, ReasonPhrase}=CodeAndReason}, Req) + when is_integer(Code) -> {Reply, NewState} = case Code of 200 -> @@ -279,11 +283,11 @@ call({set_resp_body, Body}, {?MODULE, ReqState}) -> {ok, ReqState#wm_reqstate{reqdata=wrq:set_resp_body(Body, ReqState#wm_reqstate.reqdata)}}; call(has_resp_body, {?MODULE, ReqState}) -> - Reply = case wrq:resp_body(ReqState#wm_reqstate.reqdata) of - undefined -> false; - <<>> -> false; - _ -> true - end, + Reply = + case wrq:resp_body(ReqState#wm_reqstate.reqdata) of + <<>> -> false; + _ -> true + end, {Reply, ReqState}; call({get_metadata, Key}, {?MODULE, ReqState}) -> Reply = case orddict:find(Key, ReqState#wm_reqstate.metadata) of @@ -640,7 +644,7 @@ parts_to_body(BodyList, Size, Req) when is_list(BodyList) -> {CT, _} -> CT end, - Boundary = mochihex:to_hex(crypto:rand_bytes(8)), + Boundary = mochihex:to_hex(crypto:strong_rand_bytes(8)), HeaderList = [{"Content-Type", ["multipart/byteranges; ", "boundary=", Boundary]}], @@ -720,28 +724,30 @@ make_version({1, 0}) -> make_version(_) -> <<"HTTP/1.1 ">>. +-spec update_header_with_content_length(number(), atom(), term()) -> term(). +update_header_with_content_length(Code, _Length, RD) when (Code >= 100 andalso Code < 200) orelse + Code =:= 204 orelse + Code =:= 304 -> + mochiweb_headers:make(wrq:resp_headers(RD)); +update_header_with_content_length(_Code, Length, RD) -> + case Length of + chunked -> + mochiweb_headers:enter( + "Transfer-Encoding","chunked", + mochiweb_headers:make(wrq:resp_headers(RD))); + _ -> + mochiweb_headers:enter( + "Content-Length",integer_to_list(Length), + mochiweb_headers:make(wrq:resp_headers(RD))) + end. + make_headers({Code, _ReasonPhrase}, Length, RD) -> make_headers(Code, Length, RD); make_headers(Code, Length, RD) when is_integer(Code) -> - Hdrs0 = case Code of - 304 -> - mochiweb_headers:make(wrq:resp_headers(RD)); - _ -> - case Length of - chunked -> - mochiweb_headers:enter( - "Transfer-Encoding","chunked", - mochiweb_headers:make(wrq:resp_headers(RD))); - _ -> - mochiweb_headers:enter( - "Content-Length",integer_to_list(Length), - mochiweb_headers:make(wrq:resp_headers(RD))) - end - end, - case application:get_env(webmachine, server_name) of - undefined -> ServerHeader = "MochiWeb/1.1 WebMachine/" ++ ?WMVSN ++ " (" ++ ?QUIP ++ ")"; - {ok, ServerHeader} when is_list(ServerHeader) -> ok - end, + Hdrs0 = update_header_with_content_length(Code, Length, RD), + %% server_name is guaranteed to be set by + %% webmachine_app:load_default_app_config/0 + {ok, ServerHeader} = application:get_env(webmachine, server_name), WithSrv = mochiweb_headers:enter("Server", ServerHeader, Hdrs0), Hdrs = case mochiweb_headers:get_value("date", WithSrv) of undefined -> @@ -939,6 +945,12 @@ header_test() -> ?assertEqual({HdrValue, ReqState}, get_header_value(HdrName, ReqState)), ?assertEqual({HdrValue, ReqState}, get_req_header(HdrName, ReqState)). +no_content_length_test() -> + ReqData = #wm_reqdata{req_headers = mochiweb_headers:make([])}, + ?assertEqual(nomatch, re:run(make_headers(100, 56, ReqData), "content-length", [caseless])), + ?assertEqual(nomatch, re:run(make_headers(204, 56, ReqData), "content-length", [caseless])), + ?assertMatch({match, _}, re:run(make_headers(200, 56, ReqData), "content-length", [caseless])). + metadata_test() -> Key = "webmachine", Value = "eunit", diff --git a/src/webmachine_resource.erl b/src/webmachine_resource.erl index 603dbfcb..e9a072cb 100644 --- a/src/webmachine_resource.erl +++ b/src/webmachine_resource.erl @@ -17,24 +17,40 @@ -module(webmachine_resource). -author('Justin Sheehy <justin@basho.com>'). -author('Andy Gross <andy@basho.com>'). --export([new/4, wrap/2, wrap/3]). +-export([new/3, wrap/2]). -export([do/3,log_d/2,stop/1]). +-include("wm_compat.hrl"). -include("wm_resource.hrl"). -include("wm_reqdata.hrl"). -include("wm_reqstate.hrl"). -new(R_Mod, R_ModState, R_ModExports, R_Trace) -> - {?MODULE, R_Mod, R_ModState, R_ModExports, R_Trace}. +-type t() :: #wm_resource{}. +-export_type([t/0]). + +-define(CALLBACK_ARITY, 2). + +%% Suppress Erlang/OTP 21 warnings about the new method to retrieve +%% stacktraces. +-ifdef(OTP_RELEASE). +-compile({nowarn_deprecated_function, [{erlang, get_stacktrace, 0}]}). +-endif. + +new(R_Mod, R_ModState, R_Trace) -> + case erlang:module_loaded(R_Mod) of + false -> code:ensure_loaded(R_Mod); + true -> ok + end, + #wm_resource{ + module = R_Mod, + modstate = R_ModState, + trace = R_Trace + }. -default(ping) -> - no_default; default(service_available) -> true; default(resource_exists) -> true; -default(auth_required) -> - true; default(is_authorized) -> true; default(forbidden) -> @@ -109,30 +125,29 @@ default(validate_content_checksum) -> default(_) -> no_default. -wrap(Mod, Args, {?MODULE, _, _, _, _}) -> - wrap(Mod, Args). - +-spec wrap(module(), [any()]) -> + {ok, t()} | {stop, bad_init_arg}. wrap(Mod, Args) -> case Mod:init(Args) of {ok, ModState} -> - {ok, webmachine_resource:new(Mod, ModState, - orddict:from_list(Mod:module_info(exports)), false)}; + {ok, webmachine_resource:new(Mod, ModState, false)}; {{trace, Dir}, ModState} -> {ok, File} = open_log_file(Dir, Mod), log_decision(File, v3b14), log_call(File, attempt, Mod, init, Args), log_call(File, result, Mod, init, {{trace, Dir}, ModState}), - {ok, webmachine_resource:new(Mod, ModState, - orddict:from_list(Mod:module_info(exports)), File)}; + {ok, webmachine_resource:new(Mod, ModState, File)}; _ -> {stop, bad_init_arg} end. do(#wm_resource{}=Res, Fun, ReqProps) -> - #wm_resource{module=R_Mod, modstate=R_ModState, - modexports=R_ModExports, trace=R_Trace} = Res, - do(Fun, ReqProps, {?MODULE, R_Mod, R_ModState, R_ModExports, R_Trace}); -do(Fun, ReqProps, {?MODULE, R_Mod, _, R_ModExports, R_Trace}=Req) + do(Fun, ReqProps, Res); +do(Fun, ReqProps, + #wm_resource{ + module=R_Mod, + trace=R_Trace + }=Req) when is_atom(Fun) andalso is_list(ReqProps) -> case lists:keyfind(reqstate, 1, ReqProps) of false -> RState0 = undefined; @@ -149,15 +164,20 @@ do(Fun, ReqProps, {?MODULE, R_Mod, _, R_ModExports, R_Trace}=Req) %% Do not need the embedded state anymore TrimData = ReqData#wm_reqdata{wm_state=undefined}, {Reply, - webmachine_resource:new(R_Mod, NewModState, R_ModExports, R_Trace), + webmachine_resource:new(R_Mod, NewModState, R_Trace), ReqState#wm_reqstate{reqdata=TrimData}}. -handle_wm_call(Fun, ReqData, {?MODULE,R_Mod,R_ModState,R_ModExports,R_Trace}=Req) -> +handle_wm_call(Fun, ReqData, + #wm_resource{ + module=R_Mod, + modstate=R_ModState, + trace=R_Trace + }=Req) -> case default(Fun) of no_default -> resource_call(Fun, ReqData, Req); Default -> - case orddict:is_key(Fun, R_ModExports) of + case erlang:function_exported(R_Mod, Fun, ?CALLBACK_ARITY) of true -> resource_call(Fun, ReqData, Req); false -> @@ -177,35 +197,41 @@ trim_trace([{M,F,[RD = #wm_reqdata{},S],_}|STRest]) -> [{M,F,[TrimRD,S]}|STRest]; trim_trace(X) -> X. -resource_call(F, ReqData, {?MODULE, R_Mod, R_ModState, _, R_Trace}) -> +resource_call(F, ReqData, + #wm_resource{ + module=R_Mod, + modstate=R_ModState, + trace=R_Trace + }) -> case R_Trace of false -> nop; _ -> log_call(R_Trace, attempt, R_Mod, F, [ReqData, R_ModState]) end, Result = try + %% Note: the argument list must match the definition of CALLBACK_ARITY apply(R_Mod, F, [ReqData, R_ModState]) - catch C:R -> - Reason = {C, R, trim_trace(erlang:get_stacktrace())}, + catch ?STPATTERN(C:R) -> + Reason = {C, R, trim_trace(?STACKTRACE)}, {{error, Reason}, ReqData, R_ModState} end, - case R_Trace of + case R_Trace of false -> nop; _ -> log_call(R_Trace, result, R_Mod, F, Result) end, Result. log_d(#wm_resource{}=Res, DecisionID) -> - #wm_resource{module=R_Mod, modstate=R_ModState, - modexports=R_ModExports, trace=R_Trace} = Res, - log_d(DecisionID, {?MODULE, R_Mod, R_ModState, R_ModExports, R_Trace}); -log_d(DecisionID, {?MODULE, _, _, _, R_Trace}) -> + log_d(DecisionID, Res); +log_d(DecisionID, + #wm_resource{ + trace=R_Trace + }) -> case R_Trace of false -> nop; _ -> log_decision(R_Trace, DecisionID) end. -stop(#wm_resource{trace=R_Trace}) -> close_log_file(R_Trace); -stop({?MODULE, _, _, _, R_Trace}) -> close_log_file(R_Trace). +stop(#wm_resource{trace=R_Trace}) -> close_log_file(R_Trace). log_call(File, Type, M, F, Data) -> io:format(File, diff --git a/src/webmachine_util.erl b/src/webmachine_util.erl index 30e7273f..18a57720 100644 --- a/src/webmachine_util.erl +++ b/src/webmachine_util.erl @@ -515,11 +515,11 @@ guess_mime_test() -> ".gz",".tar",".tgz"], ImgTypes = [".jpg",".jpeg",".gif",".png",".ico",".svg"], ?assertEqual([], [ T || T <- TextTypes, - 1 /= string:str(guess_mime(T),"text/") ]), + 1 /= string:str(guess_mime("file" ++ T),"text/") ]), ?assertEqual([], [ T || T <- AppTypes, - 1 /= string:str(guess_mime(T),"application/") ]), + 1 /= string:str(guess_mime("file" ++ T),"application/") ]), ?assertEqual([], [ T || T <- ImgTypes, - 1 /= string:str(guess_mime(T),"image/") ]). + 1 /= string:str(guess_mime("file" ++ T),"image/") ]). now_diff_milliseconds_test() -> diff --git a/src/wmtrace_resource.erl b/src/wmtrace_resource.erl index 021a4108..a978d6ba 100755 --- a/src/wmtrace_resource.erl +++ b/src/wmtrace_resource.erl @@ -5,8 +5,7 @@ -export([add_dispatch_rule/2, remove_dispatch_rules/0]). --export([ping/2, - init/1, +-export([init/1, resource_exists/2, content_types_provided/2, produce_html/2, @@ -51,9 +50,6 @@ remove_dispatch_rules() -> %% Resource %% -ping(ReqData, State) -> - {pong, ReqData, State}. - init(Config) -> {trace_dir, TraceDir} = proplists:lookup(trace_dir, Config), {trace_dir_exists, true} = {trace_dir_exists, filelib:is_dir(TraceDir)}, @@ -243,11 +239,12 @@ aggregate_trace_part({result, Module, Function, Result}, [{Decision,[{Module, Function, Args, Result}|Calls]}|Acc]}; aggregate_trace_part({not_exported, Module, Function, Args}, {Q, R, [{Decision,Calls}|Acc]}) -> - {Q, maybe_extract_response(Function, Args, R), + {maybe_extract_request(Function, Args, Q), + maybe_extract_response(Function, Args, R), [{Decision,[{Module, Function, Args, wmtrace_not_exported}|Calls]} |Acc]}. -maybe_extract_request(ping, [ReqData,_], _) -> +maybe_extract_request(service_available, [ReqData, _], _) -> ReqData; maybe_extract_request(_, _, R) -> R. diff --git a/src/wrq.erl b/src/wrq.erl index c7b0d624..8d9eee55 100644 --- a/src/wrq.erl +++ b/src/wrq.erl @@ -34,31 +34,47 @@ % @type reqdata(). The opaque data type used for req/resp data structures. -include("wm_reqdata.hrl"). -include("wm_reqstate.hrl"). - - +-type t() :: #wm_reqdata{}. +-export_type([t/0]). + +-type scheme() :: http | https. +-type method() ::'OPTIONS' % from erlang:decode_packet/3 + | 'GET' + | 'HEAD' + | 'POST' + | 'PUT' + | 'DELETE' + | 'TRACE' + | string() + | binary (). +-type version() :: {non_neg_integer(), non_neg_integer()}. + +-export_type([scheme/0, method/0, version/0]). + +-spec create(method(), + version(), + string(), + webmachine:headers()) -> + t(). create(Method,Version,RawPath,Headers) -> - create(Method,http,Version,RawPath,Headers). + create(Method,http,Version,RawPath,Headers). +-spec create(method(), + scheme(), + version(), + string(), + webmachine:headers()) -> + t(). create(Method,Scheme,Version,RawPath,Headers) -> - create(#wm_reqdata{method=Method,scheme=Scheme,version=Version, - raw_path=RawPath,req_headers=Headers, - wm_state=defined_on_call, - path="defined_in_create", - req_cookie=defined_in_create, - req_qs=defined_in_create, - peer="defined_in_wm_req_srv_init", - sock="defined_in_wm_req_srv_init", - req_body=not_fetched_yet, - max_recv_body=(1024*(1024*1024)), - % Stolen from R13B03 inet_drv.c's TCP_MAX_PACKET_SIZE definition - max_recv_hunk=(64*(1024*1024)), - app_root="defined_in_load_dispatch_data", - path_info=orddict:new(), - path_tokens=defined_in_load_dispatch_data, - disp_path=defined_in_load_dispatch_data, - resp_redirect=false, resp_headers=mochiweb_headers:empty(), - resp_body = <<>>, response_code=500, - resp_range = follow_request, - notes=[]}). + create( + #wm_reqdata{ + method=Method, + scheme=Scheme, + version=Version, + raw_path=RawPath, + req_headers=Headers + }). + +-spec create(t()) -> t(). create(RD = #wm_reqdata{raw_path=RawPath}) -> {Path, _, _} = mochiweb_util:urlsplit_path(RawPath), Cookie = case get_req_header("cookie", RD) of @@ -68,16 +84,20 @@ create(RD = #wm_reqdata{raw_path=RawPath}) -> {_, QueryString, _} = mochiweb_util:urlsplit_path(RawPath), ReqQS = mochiweb_util:parse_qs(QueryString), RD#wm_reqdata{path=Path,req_cookie=Cookie,req_qs=ReqQS}. + load_dispatch_data(PathInfo, HostTokens, Port, PathTokens, AppRoot, DispPath, RD) -> RD#wm_reqdata{path_info=PathInfo,host_tokens=HostTokens, port=Port,path_tokens=PathTokens, app_root=AppRoot,disp_path=DispPath}. +-spec method(t()) -> method(). method(_RD = #wm_reqdata{method=Method}) -> Method. +-spec scheme(t()) -> scheme(). scheme(_RD = #wm_reqdata{scheme=Scheme}) -> Scheme. +-spec version(t()) -> version(). version(_RD = #wm_reqdata{version=Version}) when is_tuple(Version), size(Version) == 2, is_integer(element(1,Version)), is_integer(element(2,Version)) -> Version. @@ -88,11 +108,13 @@ sock(_RD = #wm_reqdata{sock=Sock}) when is_list(Sock) -> Sock. app_root(_RD = #wm_reqdata{app_root=AR}) when is_list(AR) -> AR. -% all three paths below are strings +-spec disp_path(t()) -> string(). disp_path(_RD = #wm_reqdata{disp_path=DP}) when is_list(DP) -> DP. +-spec path(t()) -> string(). path(_RD = #wm_reqdata{path=Path}) when is_list(Path) -> Path. +-spec raw_path(t()) -> string(). raw_path(_RD = #wm_reqdata{raw_path=RawPath}) when is_list(RawPath) -> RawPath. path_info(_RD = #wm_reqdata{path_info=PathInfo}) -> PathInfo. % dict @@ -101,27 +123,33 @@ path_tokens(_RD = #wm_reqdata{path_tokens=PathT}) -> PathT. % list of strings host_tokens(_RD = #wm_reqdata{host_tokens=HostT}) -> HostT. % list of strings -port(_RD = #wm_reqdata{port=Port}) -> Port. % integer +-spec port(t()) -> inet:port_number(). +port(_RD = #wm_reqdata{port=Port}) -> Port. -response_code(_RD = #wm_reqdata{response_code={C,_ReasonPhrase}}) when is_integer(C) -> C; -response_code(_RD = #wm_reqdata{response_code=C}) when is_integer(C) -> C. +-spec response_code(t()) -> non_neg_integer(). +response_code(#wm_reqdata{response_code={C,_ReasonPhrase}}) + when is_integer(C) -> C; +response_code(_RD = #wm_reqdata{response_code=C}) + when is_integer(C) -> C. -req_cookie(_RD = #wm_reqdata{req_cookie=C}) when is_list(C) -> C. % string +-spec req_cookie(t()) -> [{string(), string()}]. +req_cookie(_RD = #wm_reqdata{req_cookie=C}) when is_list(C) -> C. -%% @spec req_qs(reqdata()) -> [{Key, Value}] +-spec req_qs(t()) -> [{string(), string()}]. req_qs(_RD = #wm_reqdata{req_qs=QS}) when is_list(QS) -> QS. -req_headers(_RD = #wm_reqdata{req_headers=ReqH}) -> ReqH. % mochiheaders +-spec req_headers(t()) -> webmachine:headers(). +req_headers(_RD = #wm_reqdata{req_headers=ReqH}) -> ReqH. req_body(_RD = #wm_reqdata{wm_state=ReqState0,max_recv_body=MRB}) -> Req = webmachine_request:new(ReqState0), - {ReqResp, ReqState} = Req:req_body(MRB), + {ReqResp, ReqState} = webmachine_request:req_body(MRB, Req), put(tmp_reqstate, ReqState), maybe_conflict_body(ReqResp). stream_req_body(_RD = #wm_reqdata{wm_state=ReqState0}, MaxHunk) -> Req = webmachine_request:new(ReqState0), - {ReqResp, ReqState} = Req:stream_req_body(MaxHunk), + {ReqResp, ReqState} = webmachine_request:stream_req_body(MaxHunk, Req), put(tmp_reqstate, ReqState), maybe_conflict_body(ReqResp). @@ -139,12 +167,13 @@ maybe_conflict_body(BodyResponse) -> BodyResponse end. -resp_redirect(_RD = #wm_reqdata{resp_redirect=true}) -> true; -resp_redirect(_RD = #wm_reqdata{resp_redirect=false}) -> false. +-spec resp_redirect(t()) -> boolean(). +resp_redirect(#wm_reqdata{resp_redirect=R}) -> R. +-spec resp_headers(t()) -> webmachine_headers:headers(). resp_headers(_RD = #wm_reqdata{resp_headers=RespH}) -> RespH. % mochiheaders -resp_body(_RD = #wm_reqdata{resp_body=undefined}) -> undefined; +-spec resp_body(t()) -> webmachine:response_body(). resp_body(_RD = #wm_reqdata{resp_body={stream,X}}) -> {stream,X}; resp_body(_RD = #wm_reqdata{resp_body={known_length_stream,X,Y}}) -> {known_length_stream,X,Y}; resp_body(_RD = #wm_reqdata{resp_body={stream,X,Y}}) -> {stream,X,Y}; @@ -162,11 +191,13 @@ path_info(Key, RD) when is_atom(Key) -> error -> undefined end. +-spec get_req_header(webmachine_headers:name(), t()) -> + undefined | webmachine_headers:value(). get_req_header(HdrName, RD) -> % string->string mochiweb_headers:get_value(HdrName, req_headers(RD)). -do_redirect(true, RD) -> RD#wm_reqdata{resp_redirect=true}; -do_redirect(false, RD) -> RD#wm_reqdata{resp_redirect=false}. +-spec do_redirect(boolean(), t()) -> t(). +do_redirect(Bool, RD) -> RD#wm_reqdata{resp_redirect=Bool}. set_peer(P, RD) when is_list(P) -> RD#wm_reqdata{peer=P}. % string @@ -185,15 +216,19 @@ set_response_code(Code, RD) when is_integer(Code) -> get_resp_header(HdrName, _RD=#wm_reqdata{resp_headers=RespH}) -> mochiweb_headers:get_value(HdrName, RespH). + set_resp_header(K, V, RD=#wm_reqdata{resp_headers=RespH}) when is_list(K),is_list(V) -> RD#wm_reqdata{resp_headers=mochiweb_headers:enter(K, V, RespH)}. + set_resp_headers(Hdrs, RD=#wm_reqdata{resp_headers=RespH}) -> F = fun({K, V}, Acc) -> mochiweb_headers:enter(K, V, Acc) end, RD#wm_reqdata{resp_headers=lists:foldl(F, RespH, Hdrs)}. + fresh_resp_headers(Hdrs, RD) -> F = fun({K, V}, Acc) -> mochiweb_headers:enter(K, V, Acc) end, RD#wm_reqdata{resp_headers=lists:foldl(F, mochiweb_headers:empty(), Hdrs)}. + remove_resp_header(K, RD=#wm_reqdata{resp_headers=RespH}) when is_list(K) -> RD#wm_reqdata{resp_headers=mochiweb_headers:from_list( proplists:delete(K, @@ -204,65 +239,62 @@ merge_resp_headers(Hdrs, RD=#wm_reqdata{resp_headers=RespH}) -> NewHdrs = lists:foldl(F, RespH, Hdrs), RD#wm_reqdata{resp_headers=NewHdrs}. +-spec append_to_resp_body(iolist() | binary(), t()) -> t(). append_to_resp_body(Data, RD) -> append_to_response_body(Data, RD). -append_to_response_body(Data, RD=#wm_reqdata{resp_body=RespB}) -> - case is_binary(Data) of - true -> - Data0 = RespB, - Data1 = <<Data0/binary,Data/binary>>, - RD#wm_reqdata{resp_body=Data1}; - false -> % MUST BE an iolist! else, fail. - append_to_response_body(iolist_to_binary(Data), RD) - end. --spec set_resp_range(follow_request | ignore_request, #wm_reqdata{}) -> #wm_reqdata{}. -%% follow_request : range responce for range request, normal responce for non-range one -%% ignore_request : normal resopnse for either range reuqest or non-range one +-spec append_to_response_body(iolist() | binary(), t()) -> t(). +append_to_response_body(IOList, RD) when is_list(IOList) -> + append_to_response_body(iolist_to_binary(IOList), RD); +append_to_response_body(Data, RD=#wm_reqdata{resp_body=RespB}) + when is_binary(Data) -> + Data0 = RespB, + Data1 = <<Data0/binary,Data/binary>>, + RD#wm_reqdata{resp_body=Data1}. + +-spec set_resp_range(follow_request | ignore_request, t()) -> t(). set_resp_range(RespRange, RD) when RespRange =:= follow_request orelse RespRange =:= ignore_request -> RD#wm_reqdata{resp_range = RespRange}. -get_cookie_value(Key, RD) when is_list(Key) -> % string +-spec get_cookie_value(string(), t()) -> string() | undefined. +get_cookie_value(Key, RD) when is_list(Key) -> case lists:keyfind(Key, 1, req_cookie(RD)) of false -> undefined; {Key, Value} -> Value end. +-spec get_qs_value(string(), t()) -> string() | undefined. get_qs_value(Key, RD) when is_list(Key) -> % string case lists:keyfind(Key, 1, req_qs(RD)) of false -> undefined; {Key, Value} -> Value end. +-spec get_qs_value(string(), string(), t()) -> string(). get_qs_value(Key, Default, RD) when is_list(Key) -> case lists:keyfind(Key, 1, req_qs(RD)) of false -> Default; {Key, Value} -> Value end. + +-spec add_note(any(), any(), t()) -> t(). add_note(K, V, RD) -> RD#wm_reqdata{notes=[{K, V} | RD#wm_reqdata.notes]}. +-spec get_notes(t()) -> list(). get_notes(RD) -> RD#wm_reqdata.notes. +-spec base_uri(t()) -> string(). base_uri(RD) -> Scheme = erlang:atom_to_list(RD#wm_reqdata.scheme), Host = string:join(RD#wm_reqdata.host_tokens, "."), PortString = port_string(RD#wm_reqdata.scheme, RD#wm_reqdata.port), Scheme ++ "://" ++ Host ++ PortString. -port_string(Scheme, Port) -> - case Scheme of - http -> - case Port of - 80 -> ""; - _ -> ":" ++ erlang:integer_to_list(Port) - end; - https -> - case Port of - 443 -> ""; - _ -> ":" ++ erlang:integer_to_list(Port) - end; - _ -> ":" ++ erlang:integer_to_list(Port) - end. +-spec port_string(scheme(), inet:port_number()) -> string(). +port_string(http, 80) -> ""; +port_string(https, 443) -> ""; +port_string(_, Port) -> + ":" ++ erlang:integer_to_list(Port). %% %% Tests @@ -283,19 +315,19 @@ accessor_test() -> ?assertEqual('GET', method(R)), ?assertEqual({1,1}, version(R)), ?assertEqual("/foo", path(R)), - ?assertEqual("/foo?a=1&b=2", raw_path(R)), + ?assertEqual("/foo?a=1&b=2", raw_path(R)), ?assertEqual([{"a", "1"}, {"b", "2"}], req_qs(R)), ?assertEqual({"1", "2"}, {get_qs_value("a", R), get_qs_value("b", R)}), ?assertEqual("3", get_qs_value("c", "3", R)), ?assertEqual([{"foo", "bar"}], req_cookie(R)), ?assertEqual("bar", get_cookie_value("foo", R)), ?assertEqual("127.0.0.1", peer(R)). - + simple_dispatch_test() -> R0 = make_wrq('GET', "/foo?a=1&b=2", [{"Cookie", "foo=bar"}]), - R1 = set_peer("127.0.0.1", R0), - {_, _, HostTokens, Port, PathTokens, Bindings, AppRoot, StringPath} = - webmachine_dispatcher:dispatch("127.0.0.1", "/foo", + R1 = set_peer("127.0.0.1", R0), + {_, _, HostTokens, Port, PathTokens, Bindings, AppRoot, StringPath} = + webmachine_dispatcher:dispatch("127.0.0.1", "/foo", [{["foo"], foo_resource, []}], R1), R = load_dispatch_data(Bindings, HostTokens, diff --git a/test/decision_core_test.erl b/test/decision_core_test.erl index 44fd0ee2..7f0be4bb 100644 --- a/test/decision_core_test.erl +++ b/test/decision_core_test.erl @@ -17,10 +17,20 @@ -ifdef(TEST). +-include("wm_compat.hrl"). -include("wm_reqdata.hrl"). -include_lib("eunit/include/eunit.hrl"). --compile(export_all). +-export([size_stream_raises_error/2, process_post_for_created_p11/3, get_streamed_body/2, send_streamed_body/2, + accept_text/2, writer_response/2, known_length_body/2, range_response/2, stream_content_md5/0, + validate_checksum_for_md5stream/3, process_post_for_md5_stream/3, init/1, service_available/2, + validate_content_checksum/2, is_authorized/2, allowed_methods/2, known_methods/2, uri_too_long/2, + known_content_type/2, valid_entity_length/2, malformed_request/2, forbidden/2, valid_content_headers/2, + content_types_provided/2, content_types_accepted/2, language_available/2, charsets_provided/2, + encodings_provided/2, resource_exists/2, generate_etag/2, last_modified/2, moved_permanently/2, + moved_temporarily/2, previously_existed/2, allow_missing_post/2, post_is_create/2, process_post/2, + create_path/2, is_conflict/2, multiple_choices/2, base_uri/2, base_uri_add_slash/1, expires/2, + delete_resource/2, delete_completed/2, to_html/2]). -define(RESOURCE, atom_to_list(?MODULE)). -define(RESOURCE_PATH, "/" ++ ?RESOURCE). @@ -35,6 +45,12 @@ md5(Bin) -> crypto:md5(Bin). -endif. +%% Suppress Erlang/OTP 21 warnings about the new method to retrieve +%% stacktraces. +-ifdef(OTP_RELEASE). +-compile({nowarn_deprecated_function, [{erlang, get_stacktrace, 0}]}). +-endif. + -define(HTTP_1_0_METHODS, ['GET', 'POST', 'HEAD']). -define(HTTP_1_1_METHODS, ['GET', 'HEAD', 'POST', 'PUT', 'DELETE', 'TRACE', 'CONNECT', 'OPTIONS']). @@ -240,8 +256,6 @@ md5(Bin) -> %% core_tests() -> [fun service_unavailable/0, - fun ping_invalid/0, - fun ping_error/0, fun internal_server_error_o18/0, fun not_implemented_b12/0, fun not_implemented_b6/0, @@ -305,7 +319,7 @@ core_tests() -> fun head_length_access_for_cs/0, fun get_known_length_for_cs/0, fun get_for_range_capable_stream/0 - %% known_failure -- fun stream_content_md5/0 + % known_failure -- fun stream_content_md5/0 ]. decision_core_test_() -> @@ -322,8 +336,8 @@ setup() -> meck:new(webmachine_resource, MeckOpts), Ctx catch - T:E -> - io:format(user, "~n~p : ~p : ~p", [T, E, erlang:get_stacktrace()]), + ?STPATTERN(T:E) -> + io:format(user, "~n~p : ~p : ~p", [T, E, ?STACKTRACE]), error(setup_failed) end. @@ -374,24 +388,6 @@ service_unavailable() -> ?assertEqual(ExpectedDecisionTrace, get_decision_ids()), ok. -%% 503 result via B13 (at ping) -ping_invalid() -> - % "breakout" for "anything other than pong" - put_setting(ping, breakout), - {ok, Result} = httpc:request(head, {url(), []}, [], []), - ?assertMatch({{"HTTP/1.1", 503, "Service Unavailable"}, _, _}, Result), - ExpectedDecisionTrace = ?PATH_TO_B13, - ?assertEqual(ExpectedDecisionTrace, get_decision_ids()), - ok. - -%% 500 error response result via B13 (ping raises error) -ping_error() -> - put_setting(ping, ping_raise_error), - {ok, Result} = httpc:request(head, {url(), []}, [], []), - ?assertMatch({{"HTTP/1.1", 500, "Internal Server Error"}, _, _}, Result), - ExpectedDecisionTrace = ?PATH_TO_B13, - ?assertEqual(ExpectedDecisionTrace, get_decision_ids()), - ok. %% 500 error response via O18 from a callback raising an error internal_server_error_o18() -> @@ -491,7 +487,9 @@ non_standard_method_501() -> Port = wm_integration_test_util:get_port(Ctx), Url = wm_integration_test_util:url(Ctx, "foo"), {ok, Sock} = gen_tcp:connect("localhost", Port, [binary, {active,false}]), - ok = gen_tcp:send(Sock, ["FOO ", Url, " HTTP/1.1\r\nConnection: close\r\n\r\n"]), + ok = gen_tcp:send(Sock, ["FOO ", Url, " HTTP/1.1\r\n", + "Host: http://localhost:", integer_to_list(Port), + "\r\n\r\n"]), ?assertMatch({ok, <<"HTTP/1.1 501 Not Implemented", _/binary>>}, gen_tcp:recv(Sock, 0, 2000)), ok = gen_tcp:close(Sock), @@ -506,7 +504,9 @@ non_standard_method_200() -> Port = wm_integration_test_util:get_port(Ctx), Url = wm_integration_test_util:url(Ctx, "foo"), {ok, Sock} = gen_tcp:connect("localhost", Port, [binary, {active,false}]), - ok = gen_tcp:send(Sock, [Method, " ", Url, " HTTP/1.1\r\nConnection: close\r\n\r\n"]), + ok = gen_tcp:send(Sock, [Method, " ", Url, " HTTP/1.1\r\n", + "Host: http://localhost:", integer_to_list(Port), + "\r\n\r\n"]), ?assertMatch({ok, <<"HTTP/1.1 200 OK\r\n", _/binary>>}, gen_tcp:recv(Sock, 0, 2000)), ok = gen_tcp:close(Sock), @@ -904,10 +904,15 @@ not_modified_j18_via_h12() -> %% 304 result via L17 not_modified_l17() -> + % Because httpd_util:rfc1123_date converts the date to GMT, it is offset by + % the local time zone. To get a Last-Modified date equal to the + % If-Modified-Since date we must convert it back from the RFC 1123 format. + % This avoids the test case failing in any time zone "east" of GMT :-P + RFC1123LastYear = httpd_util:rfc1123_date(?FIRST_DAY_OF_LAST_YEAR), + DateTimeLastYear = httpd_util:convert_request_date(RFC1123LastYear), put_setting(allowed_methods, ?DEFAULT_ALLOWED_METHODS), - put_setting(last_modified, ?FIRST_DAY_OF_LAST_YEAR), + put_setting(last_modified, DateTimeLastYear), put_setting(expires, ?FIRST_DAY_OF_NEXT_YEAR), - RFC1123LastYear = httpd_util:rfc1123_date(?FIRST_DAY_OF_LAST_YEAR), Headers = [{"If-Modified-Since", RFC1123LastYear}], {ok, Result} = httpc:request(get, {url(), Headers}, [], []), ?assertMatch({{"HTTP/1.1", 304, "Not Modified"}, _, _}, Result), @@ -1377,13 +1382,6 @@ url(Path) -> init([]) -> {ok, undefined}. -ping(ReqData, State) -> - Setting = lookup_setting(ping), - case Setting of - ping_raise_error -> error(foobar); - _ -> {Setting, ReqData, State} - end. - service_available(ReqData, Context) -> Setting = lookup_setting(service_available), {Setting, ReqData, Context}. diff --git a/test/wm_integration_test.erl b/test/wm_integration_test.erl index 9aac6413..1f58cd31 100644 --- a/test/wm_integration_test.erl +++ b/test/wm_integration_test.erl @@ -17,8 +17,6 @@ -include_lib("eunit/include/eunit.hrl"). -include("webmachine.hrl"). --compile([export_all]). - integration_test_() -> {foreach, %% Setup @@ -38,12 +36,22 @@ integration_test_() -> {spawn, {with, Ctx, integration_tests()}} end]}. +-ifdef(GITHUBEXCLUDE). + +integration_tests() -> + [fun test_host_header_localhost/1, + fun test_host_header_127/1]. + +-else. + integration_tests() -> [fun test_host_header_localhost/1, fun test_host_header_127/1, fun test_host_header_ipv6/1, fun test_host_header_ipv6_curl/1]. +-endif. + test_host_header_localhost(Ctx) -> ExpectHost = add_port(Ctx, "localhost"), verify_host_header(Ctx, "localhost", ExpectHost, <<"localhost">>). @@ -52,6 +60,8 @@ test_host_header_127(Ctx) -> ExpectHost = add_port(Ctx, "127.0.0.1"), verify_host_header(Ctx, "127.0.0.1", ExpectHost, <<"127.0.0.1">>). +-ifndef(GITHUBEXCLUDE). + test_host_header_ipv6(Ctx) -> %% Bare ipv6 addresses must be enclosed in square %% brackets. ibrowse does the right thing in parsing the URL, but @@ -79,6 +89,8 @@ test_host_header_ipv6_curl(Ctx) -> ?assertEqual(<<"[::1]">>, proplists:get_value(<<"HostTokens">>, Got)) end. +-endif. + url(Ctx, Host, Path) -> Port = erlang:integer_to_list(wm_integration_test_util:get_port(Ctx)), "http://" ++ Host ++ ":" ++ Port ++ slash(Path). @@ -102,4 +114,5 @@ verify_host_header(Ctx, Host, ExpectHostHeader, ExpectHostTokens) -> ?assertEqual(ExpectHostHeader, proplists:get_value(<<"Host">>, Got)), ?assertEqual(ExpectHostTokens, proplists:get_value(<<"HostTokens">>, Got)). + -endif.