Initial commit of Aaron Stone's evhtp client API

Conflicts:
	CMakeLists.txt
	evhtp.c
	evhtp.h
	htparse/htparse.h
	test.c

.gitignore

@@ -15,4 +15,6 @@
 /libevhtp.a
 /test
 /test_basic
-
+/test_vhost
+/test_client
+/test_proxy

CMakeLists.txt

@@ -185,8 +185,14 @@ endif()
 add_executable(test test.c)
 target_link_libraries(test libevhtp ${LIBEVHTP_EXTERNAL_LIBS} ${SYS_LIBS})
 
+add_executable(test_client test_client.c)
+target_link_libraries(test_client libevhtp ${LIBEVHTP_EXTERNAL_LIBS} ${SYS_LIBS})
+
 add_executable(test_basic test_basic.c)
 target_link_libraries(test_basic libevhtp ${LIBEVHTP_EXTERNAL_LIBS} ${SYS_LIBS})
 
 add_executable(test_vhost test_vhost.c)
 target_link_libraries(test_vhost libevhtp ${LIBEVHTP_EXTERNAL_LIBS} ${SYS_LIBS})
+
+add_executable(test_proxy test_proxy.c)
+target_link_libraries(test_proxy libevhtp ${LIBEVHTP_EXTERNAL_LIBS} ${SYS_LIBS})

evhtp.c

@@ -33,7 +33,7 @@ static int                  _evhtp_request_parser_headers_start(htparser * p);
 
 static void                 _evhtp_connection_readcb(evbev_t * bev, void * arg);
 
-static evhtp_connection_t * _evhtp_connection_new(evhtp_t * htp, int sock);
+static evhtp_connection_t * _evhtp_connection_new(evhtp_t * htp, int sock, evhtp_type type);
 
 static evhtp_uri_t        * _evhtp_uri_new(void);
 static void                 _evhtp_uri_free(evhtp_uri_t * uri);
@@ -722,7 +722,7 @@ _evhtp_request_new(evhtp_connection_t * c) {
     }
 
     req->conn        = c;
-    req->htp         = c->htp;
+    req->htp         = c ? c->htp : NULL;
     req->status      = EVHTP_RES_OK;
     req->buffer_in   = evbuffer_new();
     req->buffer_out  = evbuffer_new();
@@ -944,6 +944,10 @@ static int
 _evhtp_request_parser_start(htparser * p) {
     evhtp_connection_t * c = htparser_get_userdata(p);
 
+    if (c->type == evhtp_type_client) {
+        return 0;
+    }
+
     if (c->request) {
         if (c->request->finished == 1) {
             _evhtp_request_free(c->request);
@@ -964,6 +968,13 @@ _evhtp_request_parser_args(htparser * p, const char * data, size_t len) {
     evhtp_connection_t * c   = htparser_get_userdata(p);
     evhtp_uri_t        * uri = c->request->uri;
 
+    if (c->type == evhtp_type_client) {
+        /* as a client, technically we should never get here, but just in case
+         * we return a 0 to the parser to continue.
+         */
+        return 0;
+    }
+
     if (!(uri->query = evhtp_parse_query(data, len))) {
         c->request->status = EVHTP_RES_ERROR;
         return -1;
@@ -1522,14 +1533,19 @@ _evhtp_connection_writecb(evbev_t * bev, void * arg) {
 
 static void
 _evhtp_connection_eventcb(evbev_t * bev, short events, void * arg) {
-    evhtp_connection_t * c;
+    evhtp_connection_t * c = arg;
 
     if ((events & BEV_EVENT_CONNECTED)) {
+        if (c->type == evhtp_type_client) {
+            bufferevent_setcb(bev,
+                              _evhtp_connection_readcb,
+                              _evhtp_connection_writecb,
+                              _evhtp_connection_eventcb, c);
+        }
+
         return;
     }
 
-    c = arg;
-
     if (c->ssl && !(events & BEV_EVENT_EOF)) {
         /* XXX need to do better error handling for SSL specific errors */
         c->error = 1;
@@ -1631,22 +1647,44 @@ _evhtp_default_request_cb(evhtp_request_t * request, void * arg) {
 }
 
 static evhtp_connection_t *
-_evhtp_connection_new(evhtp_t * htp, int sock) {
+_evhtp_connection_new(evhtp_t * htp, int sock, evhtp_type type) {
     evhtp_connection_t * connection;
+    htp_type             ptype;
+
+    switch (type) {
+        case evhtp_type_client:
+            ptype = htp_type_response;
+            break;
+        case evhtp_type_server:
+            ptype = htp_type_request;
+            break;
+        default:
+            return NULL;
+    }
 
     if (!(connection = calloc(sizeof(evhtp_connection_t), 1))) {
         return NULL;
     }
 
-    connection->error  = 0;
-    connection->owner  = 1;
-    connection->sock   = sock;
-    connection->htp    = htp;
-    connection->parser = htparser_new();
+    connection->evbase    = NULL;
+    connection->bev       = NULL;
+    connection->thread    = NULL;
+    connection->hooks     = NULL;
+    connection->request   = NULL;
+    connection->resume_ev = NULL;
+    connection->saddr     = NULL;
+    connection->error     = 0;
+    connection->owner     = 1;
+    connection->sock      = sock;
+    connection->htp       = htp;
+    connection->type      = type;
+    connection->parser    = htparser_new();
 
-    htparser_init(connection->parser, htp_type_request);
+    htparser_init(connection->parser, ptype);
     htparser_set_userdata(connection->parser, connection);
 
+    TAILQ_INIT(&connection->pending);
+
     return connection;
 }
 
@@ -1705,7 +1743,7 @@ _evhtp_accept_cb(evserv_t * serv, int fd, struct sockaddr * s, int sl, void * ar
     evhtp_t            * htp = arg;
     evhtp_connection_t * connection;
 
-    if (!(connection = _evhtp_connection_new(htp, fd))) {
+    if (!(connection = _evhtp_connection_new(htp, fd, evhtp_type_server))) {
         return;
     }
 
@@ -1723,6 +1761,10 @@ _evhtp_accept_cb(evserv_t * serv, int fd, struct sockaddr * s, int sl, void * ar
 #endif
     connection->evbase = htp->evbase;
 
+    if (_evhtp_run_pre_accept(connection->htp, connection) < 0) {
+        return evhtp_connection_free(connection);
+    }
+
     if (_evhtp_connection_accept(htp->evbase, connection) < 0) {
         return evhtp_connection_free(connection);
     }
@@ -2096,6 +2138,19 @@ evhtp_kvs_add_kv(evhtp_kvs_t * kvs, evhtp_kv_t * kv) {
     TAILQ_INSERT_TAIL(kvs, kv, next);
 }
 
+void
+evhtp_kvs_add_kvs(evhtp_kvs_t * dst, evhtp_kvs_t * src) {
+    if (dst == NULL || src == NULL) {
+        return;
+    }
+
+    evhtp_kv_t * kv;
+
+    TAILQ_FOREACH(kv, src, next) {
+        evhtp_kvs_add_kv(dst, evhtp_kv_new(kv->key, kv->val, kv->k_heaped, kv->v_heaped));
+    }
+}
+
 typedef enum {
     s_query_start = 0,
     s_query_question_mark,
@@ -3465,3 +3520,82 @@ evhtp_new(evbase_t * evbase, void * arg) {
     return htp;
 }
 
+/*****************************************************************
+* client request functions                                      *
+*****************************************************************/
+
+evhtp_connection_t *
+evhtp_connection_new(evbase_t * evbase, const char * addr, uint16_t port) {
+    evhtp_connection_t * conn;
+    struct sockaddr_in   sin;
+
+    if (evbase == NULL) {
+        return NULL;
+    }
+
+    if (!(conn = _evhtp_connection_new(NULL, -1, evhtp_type_client))) {
+        return NULL;
+    }
+
+    sin.sin_family      = AF_INET;
+    sin.sin_addr.s_addr = inet_addr(addr);
+    sin.sin_port        = htons(port);
+
+    conn->evbase        = evbase;
+    conn->bev           = bufferevent_socket_new(evbase, -1, BEV_OPT_CLOSE_ON_FREE);
+
+    bufferevent_enable(conn->bev, EV_READ);
+
+    bufferevent_setcb(conn->bev, NULL, NULL,
+                      _evhtp_connection_eventcb, conn);
+
+    bufferevent_socket_connect(conn->bev,
+                               (struct sockaddr *)&sin, sizeof(sin));
+
+    return conn;
+}
+
+evhtp_request_t *
+evhtp_request_new(evhtp_callback_cb cb, void * arg) {
+    evhtp_request_t * r;
+
+    if (!(r = _evhtp_request_new(NULL))) {
+        return NULL;
+    }
+
+    r->cb    = cb;
+    r->cbarg = arg;
+    r->proto = EVHTP_PROTO_11;
+
+    return r;
+}
+
+int
+evhtp_make_request(evhtp_connection_t * c, evhtp_request_t * r,
+                   htp_method meth, const char * uri) {
+    evbuf_t * obuf;
+    char    * proto;
+
+    obuf       = bufferevent_get_output(c->bev);
+    r->conn    = c;
+    c->request = r;
+
+    switch (r->proto) {
+        case EVHTP_PROTO_10:
+            proto = "1.0";
+            break;
+        case EVHTP_PROTO_11:
+        default:
+            proto = "1.1";
+            break;
+    }
+
+    evbuffer_add_printf(obuf, "%s %s HTTP/%s\r\n",
+                        htparser_get_methodstr_m(meth), uri, proto);
+
+    evhtp_headers_for_each(r->headers_out, _evhtp_create_headers, obuf);
+    evbuffer_add_reference(obuf, "\r\n", 2, NULL, NULL);
+
+    return 0;
+}
+

evhtp.h

@@ -126,6 +126,7 @@ typedef enum evhtp_hook_type       evhtp_hook_type;
 typedef enum evhtp_callback_type   evhtp_callback_type;
 typedef enum evhtp_proto           evhtp_proto;
 typedef enum evhtp_ssl_scache_type evhtp_ssl_scache_type;
+typedef enum evhtp_type            evhtp_type;
 
 typedef void (*evhtp_thread_init_cb)(evhtp_t * htp, evthr_t * thr, void * arg);
 typedef void (*evhtp_callback_cb)(evhtp_request_t * req, void * arg);
@@ -224,6 +225,11 @@ typedef void * (*evhtp_ssl_scache_init)(evhtp_t *);
 #define EVHTP_RES_VERNSUPPORT  505
 #define EVHTP_RES_BWEXEED      509
 
+enum evhtp_type {
+    evhtp_type_client,
+    evhtp_type_server
+};
+
 struct evhtp_defaults_s {
     evhtp_callback_cb    cb;
     evhtp_pre_accept_cb  pre_accept;
@@ -409,6 +415,8 @@ struct evhtp_request_s {
     evhtp_callback_cb cb;             /**< the function to call when fully processed */
     void            * cbarg;          /**< argument which is passed to the cb function */
     int               error;
+
+    TAILQ_ENTRY(evhtp_request_s) next;
 };
 
 #define evhtp_request_content_len(r) htparser_get_content_length(r->conn->parser)
@@ -423,13 +431,16 @@ struct evhtp_connection_s {
     htparser        * parser;
     event_t         * resume_ev;
     struct sockaddr * saddr;
-    struct timeval    recv_timeo;    /**< conn read timeouts (overrides global) */
-    struct timeval    send_timeo;    /**< conn write timeouts (overrides global) */
+    struct timeval    recv_timeo;                            /**< conn read timeouts (overrides global) */
+    struct timeval    send_timeo;                            /**< conn write timeouts (overrides global) */
     int               sock;
     uint8_t           error;
-    uint8_t           owner;         /**< set to 1 if this structure owns the bufferevent */
-    uint8_t           vhost_via_sni; /**< set to 1 if the vhost was found via SSL SNI */
-    evhtp_request_t * request;       /**< the request currently being processed */
+    uint8_t           owner;                                 /**< set to 1 if this structure owns the bufferevent */
+    uint8_t           vhost_via_sni;                         /**< set to 1 if the vhost was found via SSL SNI */
+    evhtp_request_t * request;                               /**< the request currently being processed */
+    evhtp_type        type;                                  /**< server or client */
+
+    TAILQ_HEAD(, evhtp_request_s) pending;                   /**< client pending data */
 };
 
 struct evhtp_hooks_s {
@@ -816,6 +827,14 @@ evhtp_kv_t  * evhtp_kvs_find_kv(evhtp_kvs_t * kvs, const char * key);
  */
 void evhtp_kvs_add_kv(evhtp_kvs_t * kvs, evhtp_kv_t * kv);
 
+/**
+ * @brief appends all key/val structures from src tailq onto dst tailq
+ *
+ * @param dst an evhtp_kvs_t structure
+ * @param src an evhtp_kvs_t structure
+ */
+void evhtp_kvs_add_kvs(evhtp_kvs_t * dst, evhtp_kvs_t * src);
+
 int  evhtp_kvs_for_each(evhtp_kvs_t * kvs, evhtp_kvs_iterator cb, void * arg);
 
 /**
@@ -906,6 +925,7 @@ const char * evhtp_header_find(evhtp_headers_t * headers, const char * key);
 #define evhtp_headers_free        evhtp_kvs_free
 #define evhtp_header_rm_and_free  evhtp_kv_rm_and_free
 #define evhtp_headers_add_header  evhtp_kvs_add_kv
+#define evhtp_headers_add_headers evhtp_kvs_add_kvs
 #define evhtp_query_new           evhtp_kvs_new
 #define evhtp_query_free          evhtp_kvs_free
 
@@ -1006,6 +1026,25 @@ void evhtp_connection_free(evhtp_connection_t * connection);
 
 void evhtp_request_free(evhtp_request_t * request);
 
+/*****************************************************************
+* client request functions                                      *
+*****************************************************************/
+
+/**
+ * @brief allocate a new connection
+ */
+evhtp_connection_t * evhtp_connection_new(evbase_t * evbase, const char * addr, uint16_t port);
+
+/**
+ * @brief allocate a new request
+ */
+evhtp_request_t * evhtp_request_new(evhtp_callback_cb cb, void * arg);
+
+/**
+ * @brief make a client request
+ */
+int evhtp_make_request(evhtp_connection_t * c, evhtp_request_t * r, htp_method meth, const char * uri);
+
 #ifdef __cplusplus
 }
 #endif

htparse/htparse.c

@@ -367,12 +367,17 @@ htparser_get_method(htparser * p) {
 }
 
 const char *
-htparser_get_methodstr(htparser * p) {
-    if (p->method >= htp_method_UNKNOWN) {
+htparser_get_methodstr_m(htp_method meth) {
+    if (meth >= htp_method_UNKNOWN) {
         return NULL;
     }
 
-    return method_strmap[p->method];
+    return method_strmap[meth];
+}
+
+const char *
+htparser_get_methodstr(htparser * p) {
+    return htparser_get_methodstr_m(p->method);
 }
 
 void

htparse/htparse.h

@@ -91,6 +91,7 @@ int            htparser_should_keep_alive(htparser * p);
 htp_scheme     htparser_get_scheme(htparser *);
 htp_method     htparser_get_method(htparser *);
 const char   * htparser_get_methodstr(htparser *);
+const char   * htparser_get_methodstr_m(htp_method);
 void           htparser_set_major(htparser *, unsigned char);
 void           htparser_set_minor(htparser *, unsigned char);
 unsigned char  htparser_get_major(htparser *);

test_client.c

@@ -0,0 +1,74 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdint.h>
+#include <errno.h>
+#include <evhtp.h>
+
+static void
+request_cb(evhtp_request_t * req, void * arg) {
+    printf("hi %zu\n", evbuffer_get_length(req->buffer_in));
+}
+
+static evhtp_res
+print_data(evhtp_request_t * req, evbuf_t * buf, void * arg) {
+    printf("Got %zu bytes\n",
+           evbuffer_get_length(buf));
+
+    return EVHTP_RES_OK;
+}
+
+static evhtp_res
+print_new_chunk_len(evhtp_request_t * req, uint64_t len, void * arg) {
+    printf("started new chunk, %zu bytes\n", len);
+
+    return EVHTP_RES_OK;
+}
+
+static evhtp_res
+print_chunk_complete(evhtp_request_t * req, void * arg) {
+    printf("ended a single chunk\n");
+
+    return EVHTP_RES_OK;
+}
+
+static evhtp_res
+print_chunks_complete(evhtp_request_t * req, void * arg) {
+    printf("all chunks read\n");
+
+    return EVHTP_RES_OK;
+}
+
+int
+main(int argc, char ** argv) {
+    evbase_t           * evbase;
+    evhtp_connection_t * conn;
+    evhtp_request_t    * request;
+
+    evbase  = event_base_new();
+    conn    = evhtp_connection_new(evbase, "75.126.169.52", 80);
+    request = evhtp_request_new(request_cb, evbase);
+
+    evhtp_set_hook(&request->hooks, evhtp_hook_on_read, print_data, evbase);
+    evhtp_set_hook(&request->hooks, evhtp_hook_on_new_chunk,
+                   print_new_chunk_len, NULL);
+    evhtp_set_hook(&request->hooks, evhtp_hook_on_chunk_complete,
+                   print_chunk_complete, NULL);
+    evhtp_set_hook(&request->hooks, evhtp_hook_on_chunks_complete,
+                   print_chunks_complete, NULL);
+
+    evhtp_headers_add_header(request->headers_out,
+                             evhtp_header_new("Host", "ieatfood.net", 0, 0));
+    evhtp_headers_add_header(request->headers_out,
+                             evhtp_header_new("User-Agent", "libevhtp", 0, 0));
+    evhtp_headers_add_header(request->headers_out,
+                             evhtp_header_new("Connection", "close", 0, 0));
+
+    evhtp_make_request(conn, request, htp_method_GET, "/");
+
+    event_base_loop(evbase, 0);
+    event_base_free(evbase);
+
+    return 0;
+}
+

test_proxy.c

@@ -0,0 +1,108 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdint.h>
+#include <errno.h>
+#include <evhtp.h>
+
+
+int
+make_request(evbase_t * evbase,
+             evthr_t * evthr,
+             const char * const host,
+             const short port,
+             const char * const path,
+             evhtp_headers_t * headers,
+             evhtp_callback_cb cb,
+             void * arg)
+{
+    evhtp_connection_t * conn;
+    evhtp_request_t    * request;
+
+    conn    = evhtp_connection_new(evbase, host, port);
+    conn->thread = evthr;
+    request = evhtp_request_new(cb, arg);
+
+    evhtp_headers_add_header(request->headers_out,
+                             evhtp_header_new("Host", "localhost", 0, 0));
+    evhtp_headers_add_header(request->headers_out,
+                             evhtp_header_new("User-Agent", "libevhtp", 0, 0));
+    evhtp_headers_add_header(request->headers_out,
+                             evhtp_header_new("Connection", "close", 0, 0));
+
+    evhtp_headers_add_headers(request->headers_out, headers);
+
+    printf("Making backend request...\n");
+    evhtp_make_request(conn, request, htp_method_GET, path);
+    printf("Ok.\n");
+
+    return 0;
+}
+
+static void
+backend_cb(evhtp_request_t * backend_req, void * arg) {
+    evhtp_request_t * frontend_req = (evhtp_request_t *)arg;
+    evbuffer_prepend_buffer(frontend_req->buffer_out, backend_req->buffer_in);
+    evhtp_headers_add_headers(frontend_req->headers_out, backend_req->headers_in);
+
+    // char body[1024] = { '\0' };
+    // ev_ssize_t len = evbuffer_copyout(frontend_req->buffer_out, body, sizeof(body));
+    // printf("Backend %zu: %s\n", len, body);
+
+    evhtp_send_reply(frontend_req, EVHTP_RES_OK);
+    evhtp_request_resume(frontend_req);
+}
+
+static void
+frontend_cb(evhtp_request_t * req, void * arg) {
+    printf("  Received frontend request on thread %d... ", (int)evthr_get_aux(req->conn->thread));
+    evhtp_request_pause(req); // Pause the frontend request while we run the backend requests.
+    make_request(evthr_get_base(req->conn->thread), req->conn->thread, "127.0.0.1", 80, req->uri->path->full, req->headers_in, backend_cb, req);
+    printf("Ok.\n");
+}
+
+// Terminate gracefully on SIGTERM
+void
+sigterm_cb(int fd, short event, void * arg)
+{
+  evbase_t * evbase = (evbase_t *)arg;
+  struct timeval tv = {.tv_usec = 100000, .tv_sec = 0}; // 100 ms
+  event_base_loopexit(evbase, &tv);
+}
+
+void
+init_thread_cb(evhtp_t * htp, evthr_t * thr, void * arg)
+{
+    static int aux = 0;
+    printf("Spinning up a thread: %d\n", ++aux);
+    evthr_set_aux(thr, (void *)aux);
+}
+
+int
+main(int argc, char ** argv) {
+    evbase_t      * evbase = event_base_new();
+    evhtp_t       * evhtp  = evhtp_new(evbase, NULL);
+
+    evhtp_set_gencb(evhtp, frontend_cb, NULL);
+
+#ifdef USE_SSL
+    evhtp_ssl_cfg_t scfg1  = { 0 };
+
+    scfg1.pemfile  = "./server.pem";
+    scfg1.privfile = "./server.pem";
+
+    evhtp_ssl_init(evhtp, &scfg1);
+#endif
+
+    evhtp_use_threads(evhtp, init_thread_cb, 8, NULL);
+
+    struct event *ev_sigterm;
+    ev_sigterm = evsignal_new(evbase, SIGTERM, sigterm_cb, evbase);
+    evsignal_add(ev_sigterm, NULL);
+
+    evhtp_bind_socket(evhtp, "0.0.0.0", 8081, 1024);
+    event_base_loop(evbase, 0);
+
+    printf("Clean exit\n");
+    return 0;
+}