From 842dca0a6637d0bed5014d00017d06c17fc97324 Mon Sep 17 00:00:00 2001 From: devgs Date: Thu, 14 Jun 2012 12:39:40 +0300 Subject: [PATCH 1/6] support for progress status for multiple simultaneous uploads --- ngx_http_uploadprogress_module.c | 519 +++++++++++++++++++++++++++---- 1 file changed, 451 insertions(+), 68 deletions(-) diff --git a/ngx_http_uploadprogress_module.c b/ngx_http_uploadprogress_module.c index 4ef3322..7cf6d43 100644 --- a/ngx_http_uploadprogress_module.c +++ b/ngx_http_uploadprogress_module.c @@ -8,6 +8,8 @@ #include #include +#include + #define TIMER_FREQUENCY 15 * 1000 typedef enum { @@ -65,7 +67,10 @@ typedef struct { ngx_str_t content_type; ngx_array_t templates; ngx_str_t header; + ngx_str_t header_mul; + ngx_str_t jsonp_parameter; + ngx_int_t json_multiple:1; } ngx_http_uploadprogress_conf_t; typedef struct { @@ -87,6 +92,8 @@ static ngx_int_t ngx_http_uploadprogress_offset_variable(ngx_http_request_t *r, ngx_http_variable_value_t *v, uintptr_t data); static ngx_int_t ngx_http_uploadprogress_status_variable(ngx_http_request_t *r, ngx_http_variable_value_t *v, uintptr_t data); +static ngx_int_t ngx_http_uploadprogress_id_variable(ngx_http_request_t *r, + ngx_http_variable_value_t *v, uintptr_t data); static ngx_int_t ngx_http_uploadprogress_callback_variable(ngx_http_request_t *r, ngx_http_variable_value_t *v, uintptr_t data); static char* ngx_http_upload_progress_set_template(ngx_conf_t * cf, ngx_http_uploadprogress_template_t *t, ngx_str_t *source); @@ -97,6 +104,7 @@ static char* ngx_http_upload_progress_template(ngx_conf_t * cf, ngx_command_t * static char* ngx_http_upload_progress_java_output(ngx_conf_t * cf, ngx_command_t * cmd, void *conf); static char* ngx_http_upload_progress_json_output(ngx_conf_t * cf, ngx_command_t * cmd, void *conf); static char* ngx_http_upload_progress_jsonp_output(ngx_conf_t * cf, ngx_command_t * cmd, void *conf); +static char* ngx_http_upload_progress_json_multiple_output(ngx_conf_t * cf, ngx_command_t * cmd, void *conf); static void ngx_clean_old_connections(ngx_event_t * ev); static ngx_int_t ngx_http_uploadprogress_content_handler(ngx_http_request_t *r); @@ -160,6 +168,13 @@ static ngx_command_t ngx_http_uploadprogress_commands[] = { 0, NULL}, + {ngx_string("upload_progress_json_multiple_output"), + NGX_HTTP_MAIN_CONF | NGX_HTTP_SRV_CONF | NGX_HTTP_LOC_CONF | NGX_CONF_NOARGS, + ngx_http_upload_progress_json_multiple_output, + NGX_HTTP_LOC_CONF_OFFSET, + 0, + NULL}, + {ngx_string("upload_progress_header"), NGX_HTTP_MAIN_CONF | NGX_HTTP_SRV_CONF | NGX_HTTP_LOC_CONF | NGX_CONF_TAKE1, ngx_conf_set_str_slot, @@ -167,6 +182,13 @@ static ngx_command_t ngx_http_uploadprogress_commands[] = { offsetof(ngx_http_uploadprogress_conf_t, header), NULL}, + {ngx_string("upload_progress_header_mul"), + NGX_HTTP_MAIN_CONF | NGX_HTTP_SRV_CONF | NGX_HTTP_LOC_CONF | NGX_CONF_TAKE1, + ngx_conf_set_str_slot, + NGX_HTTP_LOC_CONF_OFFSET, + offsetof(ngx_http_uploadprogress_conf_t, header_mul), + NULL}, + {ngx_string("upload_progress_jsonp_parameter"), NGX_HTTP_MAIN_CONF | NGX_HTTP_SRV_CONF | NGX_HTTP_LOC_CONF | NGX_CONF_TAKE1, ngx_conf_set_str_slot, @@ -195,6 +217,10 @@ static ngx_http_variable_t ngx_http_uploadprogress_variables[] = { (uintptr_t) offsetof(ngx_http_uploadprogress_node_t, err_status), NGX_HTTP_VAR_CHANGEABLE|NGX_HTTP_VAR_NOCACHEABLE|NGX_HTTP_VAR_NOHASH, 0 }, + { ngx_string("uploadprogress_id"), NULL, ngx_http_uploadprogress_id_variable, + (uintptr_t) offsetof(ngx_http_uploadprogress_node_t, err_status), + NGX_HTTP_VAR_CHANGEABLE|NGX_HTTP_VAR_NOCACHEABLE|NGX_HTTP_VAR_NOHASH, 0 }, + { ngx_string("uploadprogress_callback"), NULL, ngx_http_uploadprogress_callback_variable, (uintptr_t) NULL, NGX_HTTP_VAR_CHANGEABLE|NGX_HTTP_VAR_NOCACHEABLE|NGX_HTTP_VAR_NOHASH, 0 }, @@ -261,9 +287,17 @@ static ngx_str_t ngx_http_uploadprogress_jsonp_defaults[] = { ngx_string("$uploadprogress_callback({ \"state\" : \"uploading\", \"received\" : $uploadprogress_received, \"size\" : $uploadprogress_length });\r\n") }; +static ngx_str_t ngx_http_uploadprogress_json_multiple_defaults[] = { + ngx_string("{ \"id\" : $uploadprogress_id, \"state\" : \"starting\" }"), + ngx_string("{ \"id\" : $uploadprogress_id, \"state\" : \"error\", \"status\" : $uploadprogress_status }"), + ngx_string("{ \"id\" : $uploadprogress_id, \"state\" : \"done\" }"), + ngx_string("{ \"id\" : $uploadprogress_id, \"state\" : \"uploading\", \"received\" : $uploadprogress_received, \"size\" : $uploadprogress_length }") +}; + static ngx_array_t ngx_http_uploadprogress_global_templates; + static ngx_str_t* get_tracking_id(ngx_http_request_t * r) { @@ -328,7 +362,7 @@ get_tracking_id(ngx_http_request_t * r) i = 1; break; } - if (len<=0) + else if (!len) break; } while(p++); @@ -355,6 +389,97 @@ get_tracking_id(ngx_http_request_t * r) return NULL; } +static ngx_str_t* +get_tracking_ids_mul(ngx_http_request_t * r) +{ + u_char *p, *start_p; + ngx_uint_t i; + ngx_list_part_t *part; + ngx_table_elt_t *header; + ngx_str_t *ret, args; + ngx_http_uploadprogress_conf_t *upcf; + + upcf = ngx_http_get_module_loc_conf(r, ngx_http_uploadprogress_module); + + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, "upload-progress: get_tracking_ids"); + + part = &r->headers_in.headers.part; + header = part->elts; + + for (i = 0; /* void */ ; i++) { + + if (i >= part->nelts) { + if (part->next == NULL) { + break; + } + + part = part->next; + header = part->elts; + i = 0; + } + + if (header[i].key.len == upcf->header_mul.len + && ngx_strncasecmp(header[i].key.data, upcf->header_mul.data, + header[i].key.len) == 0) { + ret = ngx_calloc(sizeof(ngx_str_t), r->connection->log ); + ret->data = header[i].value.data; + ret->len = header[i].value.len; + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "upload-progress: get_tracking_ids found header: %V", ret); + return ret; + } + } + + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "upload-progress: get_tracking_ids no header found"); + + /* not found, check as a request arg */ + /* it is possible the request args have not been yet created (or already released) */ + /* so let's try harder first from the request line */ + args.len = r->args.len; + args.data = r->args.data; + + if (args.len && args.data) { + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "upload-progress: get_tracking_id no header found, args found"); + i = 0; + p = args.data; + do { + ngx_uint_t len = args.len - (p - args.data); + if (len >= (upcf->header_mul.len + 1) && ngx_strncasecmp(p, upcf->header_mul.data, upcf->header_mul.len) == 0 + && p[upcf->header_mul.len] == '=') { + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "upload-progress: get_tracking_id found args: %s",p); + i = 1; + break; + } + else if (!len) + break; + } + while(p++); + + if (i) { + start_p = p += upcf->header_mul.len + 1; + while (p < args.data + args.len) { + if (*((p++) + 1) == '&') { + break; + } + } + + ret = ngx_calloc(sizeof(ngx_str_t), r->connection->log); + ret->data = start_p; + ret->len = p - start_p; + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "upload-progress: get_tracking_id found args: %V",ret); + return ret; + } + } + + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "upload-progress: get_tracking_id no id found"); + return NULL; +} + static ngx_http_uploadprogress_node_t * find_node(ngx_str_t * id, ngx_http_uploadprogress_ctx_t * ctx, ngx_log_t * log) { @@ -558,9 +683,13 @@ ngx_http_reportuploads_handler(ngx_http_request_t * r) return rc; } - /* get the tracking id if any */ - id = get_tracking_id(r); + upcf = ngx_http_get_module_loc_conf(r, ngx_http_uploadprogress_module); + /* get the tracking id if any */ + if(upcf->json_multiple) + id = get_tracking_ids_mul(r); + else + id = get_tracking_id(r); if (id == NULL) { ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, @@ -571,8 +700,6 @@ ngx_http_reportuploads_handler(ngx_http_request_t * r) ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, "reportuploads handler found id: %V", id); - upcf = ngx_http_get_module_loc_conf(r, ngx_http_uploadprogress_module); - if (upcf->shm_zone == NULL) { ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, "reportuploads no shm_zone for id: %V", id); @@ -580,32 +707,6 @@ ngx_http_reportuploads_handler(ngx_http_request_t * r) return NGX_DECLINED; } - ctx = upcf->shm_zone->data; - - /* get the original connection of the upload */ - shpool = (ngx_slab_pool_t *) upcf->shm_zone->shm.addr; - - ngx_shmtx_lock(&shpool->mutex); - - up = find_node(id, ctx, r->connection->log); - if (up != NULL) { - ngx_log_debug5(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, - "reportuploads found node: %V (rest: %uO, length: %uO, done: %ui, err_status: %ui)", id, up->rest, up->length, up->done, up->err_status); - rest = up->rest; - length = up->length; - done = up->done; - err_status = up->err_status; - found = 1; - } else { - ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, - "reportuploads not found: %V", id); - } - ngx_shmtx_unlock(&shpool->mutex); - ngx_free(id); - - /* send the output */ - r->headers_out.content_type = upcf->content_type; - /* force no-cache */ expires = r->headers_out.expires; @@ -631,7 +732,7 @@ ngx_http_reportuploads_handler(ngx_http_request_t * r) if (ngx_array_init(&r->headers_out.cache_control, r->pool, 1, sizeof(ngx_table_elt_t *)) - != NGX_OK) { + != NGX_OK) { return NGX_HTTP_INTERNAL_SERVER_ERROR; } @@ -675,59 +776,282 @@ ngx_http_reportuploads_handler(ngx_http_request_t * r) } } - ngx_http_set_ctx(r, up, ngx_http_uploadprogress_module); + ctx = upcf->shm_zone->data; + + /* get the original connection of the upload */ + shpool = (ngx_slab_pool_t *) upcf->shm_zone->shm.addr; + + if(upcf->json_multiple) + { + ngx_chain_t * p_chain_end = 0; + ngx_chain_t * p_chain_start = 0; + size_t offs = 0; + u_char * p1 = id->data, * p2; + r->headers_out.content_length_n = 0; + while(offs < id->len) + { + p2 = memchr((char *)id->data + offs, ';', id->len - offs); + if(!p2) p2 = id->data + id->len; + size_t len = p2 - p1; + if(len) + { + ngx_str_t sub_id; + sub_id.data = p1; + sub_id.len = len; + + // ----> + + ngx_shmtx_lock(&shpool->mutex); + + up = find_node(&sub_id, ctx, r->connection->log); + if (up != NULL) { + ngx_log_debug5(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "reportuploads found node: %V (rest: %uO, length: %uO, done: %ui, err_status: %ui)", &sub_id, up->rest, up->length, up->done, up->err_status); + rest = up->rest; + length = up->length; + done = up->done; + err_status = up->err_status; + found = 1; + } else { + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "reportuploads not found: %V", &sub_id); + } + ngx_shmtx_unlock(&shpool->mutex); + + /* send the output */ + r->headers_out.content_type = upcf->content_type; + + if(up == NULL) + { + // For current id + ngx_http_uploadprogress_node_t * tmp_node = ngx_pcalloc(r->pool, sizeof(ngx_http_uploadprogress_node_t) + sub_id.len); + tmp_node->len = sub_id.len; + ngx_memcpy(tmp_node->data, sub_id.data, sub_id.len); + ngx_http_set_ctx(r, tmp_node, ngx_http_uploadprogress_module); + } + else + ngx_http_set_ctx(r, up, ngx_http_uploadprogress_module); + + + if (!found) { + state = uploadprogress_state_starting; + } else if (err_status >= NGX_HTTP_BAD_REQUEST) { + state = uploadprogress_state_error; + } else if (done) { + state = uploadprogress_state_done; + } else if ( length == 0 && rest == 0 ) { + state = uploadprogress_state_starting; + } else { + state = uploadprogress_state_uploading; + } + + t = upcf->templates.elts; + + if (ngx_http_script_run(r, &response, t[(ngx_uint_t)state].lengths->elts, 0, + t[(ngx_uint_t)state].values->elts) == NULL) + { + ngx_free(id); + return NGX_HTTP_INTERNAL_SERVER_ERROR; + } + + ngx_log_debug4(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "upload progress: state=%d, err_status=%ui, remaining=%uO, length=%uO", + state, err_status, (length - rest), length); + + if(p_chain_end) + { + p_chain_end->next = ngx_palloc(r->pool, sizeof(ngx_chain_t)); + if (p_chain_end->next == NULL) { + ngx_free(id); + return NGX_HTTP_INTERNAL_SERVER_ERROR; + } + p_chain_end = p_chain_end->next; + + // Insert comma + b = ngx_calloc_buf(r->pool); + if (b == NULL) { + ngx_free(id); + return NGX_HTTP_INTERNAL_SERVER_ERROR; + } + + b->pos = b->start = ngx_palloc(r->pool, 2); + if (b->pos == NULL) { + ngx_free(id); + return NGX_HTTP_INTERNAL_SERVER_ERROR; + } + b->last = b->end = b->pos + 2; + ngx_memcpy(b->pos, ", ", 2); + b->temporary = 1; + b->memory = 1; + + p_chain_end->buf = b; + p_chain_end->next = ngx_palloc(r->pool, sizeof(ngx_chain_t)); + if (p_chain_end->next == NULL) { + ngx_free(id); + return NGX_HTTP_INTERNAL_SERVER_ERROR; + } + p_chain_end = p_chain_end->next; + } + else + { + p_chain_start = p_chain_end = ngx_palloc(r->pool, sizeof(ngx_chain_t)); + } + + b = ngx_calloc_buf(r->pool); + if (b == NULL) { + ngx_free(id); + return NGX_HTTP_INTERNAL_SERVER_ERROR; + } + + b->pos = b->start = response.data; + b->last = b->end = response.data + response.len; + + b->temporary = 1; + b->memory = 1; + + p_chain_end->buf = b; + p_chain_end->next = NULL; + + // ----> + + r->headers_out.content_length_n += b->last - b->pos; + + p1 = p2 + 1; + offs += len + 1; + } + } + ngx_free(id); + if(!p_chain_end) // Malformed id + { + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "reportuploads malformed multiple id"); + return NGX_DECLINED; + } + // Prepend brace + b = ngx_calloc_buf(r->pool); + if (b == NULL) { + ngx_free(id); + return NGX_HTTP_INTERNAL_SERVER_ERROR; + } + b->pos = b->start = ngx_palloc(r->pool, 2); + if (b->pos == NULL) { + return NGX_HTTP_INTERNAL_SERVER_ERROR; + } + b->last = b->end = b->pos + 2; + ngx_memcpy(b->pos, "[ ", 2); + b->temporary = 1; + b->memory = 1; + r->headers_out.content_length_n += 2; + + out.buf = b; + out.next = p_chain_start; + + // Append brace + p_chain_end->next = ngx_palloc(r->pool, sizeof(ngx_chain_t)); + if (p_chain_end->next == NULL) { + return NGX_HTTP_INTERNAL_SERVER_ERROR; + } + p_chain_end = p_chain_end->next; + + b = ngx_calloc_buf(r->pool); + if (b == NULL) { + return NGX_HTTP_INTERNAL_SERVER_ERROR; + } + + b->pos = b->start = ngx_palloc(r->pool, 2); + if (b->pos == NULL) { + return NGX_HTTP_INTERNAL_SERVER_ERROR; + } + b->last = b->end = b->pos + 4; + ngx_memcpy(b->pos, " ]\r\n", 4); + b->temporary = 1; + b->memory = 1; + r->headers_out.content_length_n += 4; + + p_chain_end->buf = b; + p_chain_end->next = NULL; + + r->headers_out.status = NGX_HTTP_OK; + p_chain_end->buf->last_buf = 1; + } + else + { + ngx_shmtx_lock(&shpool->mutex); + + up = find_node(id, ctx, r->connection->log); + if (up != NULL) { + ngx_log_debug5(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "reportuploads found node: %V (rest: %uO, length: %uO, done: %ui, err_status: %ui)", id, up->rest, up->length, up->done, up->err_status); + rest = up->rest; + length = up->length; + done = up->done; + err_status = up->err_status; + found = 1; + } else { + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "reportuploads not found: %V", id); + } + ngx_shmtx_unlock(&shpool->mutex); + ngx_free(id); + + /* send the output */ + r->headers_out.content_type = upcf->content_type; -/* + ngx_http_set_ctx(r, up, ngx_http_uploadprogress_module); + + /* There are 4 possibilities * request not yet started: found = false * request in error: err_status >= NGX_HTTP_BAD_REQUEST * request finished: done = true * request not yet started but registered: length==0 && rest ==0 - * reauest in progress: rest > 0 + * reauest in progress: rest > 0 */ - if (!found) { - state = uploadprogress_state_starting; - } else if (err_status >= NGX_HTTP_BAD_REQUEST) { - state = uploadprogress_state_error; - } else if (done) { - state = uploadprogress_state_done; - } else if ( length == 0 && rest == 0 ) { - state = uploadprogress_state_starting; - } else { - state = uploadprogress_state_uploading; - } + if (!found) { + state = uploadprogress_state_starting; + } else if (err_status >= NGX_HTTP_BAD_REQUEST) { + state = uploadprogress_state_error; + } else if (done) { + state = uploadprogress_state_done; + } else if ( length == 0 && rest == 0 ) { + state = uploadprogress_state_starting; + } else { + state = uploadprogress_state_uploading; + } - t = upcf->templates.elts; + t = upcf->templates.elts; - if (ngx_http_script_run(r, &response, t[(ngx_uint_t)state].lengths->elts, 0, - t[(ngx_uint_t)state].values->elts) == NULL) - { - return NGX_HTTP_INTERNAL_SERVER_ERROR; - } + if (ngx_http_script_run(r, &response, t[(ngx_uint_t)state].lengths->elts, 0, + t[(ngx_uint_t)state].values->elts) == NULL) + { + return NGX_HTTP_INTERNAL_SERVER_ERROR; + } - ngx_log_debug4(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, - "upload progress: state=%d, err_status=%ui, remaining=%uO, length=%uO", - state, err_status, (length - rest), length); + ngx_log_debug4(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "upload progress: state=%d, err_status=%ui, remaining=%uO, length=%uO", + state, err_status, (length - rest), length); - b = ngx_calloc_buf(r->pool); - if (b == NULL) { - return NGX_HTTP_INTERNAL_SERVER_ERROR; - } + b = ngx_calloc_buf(r->pool); + if (b == NULL) { + return NGX_HTTP_INTERNAL_SERVER_ERROR; + } + + b->pos = b->start = response.data; + b->last = b->end = response.data + response.len; - b->pos = b->start = response.data; - b->last = b->end = response.data + response.len; + b->temporary = 1; + b->memory = 1; - b->temporary = 1; - b->memory = 1; + out.buf = b; + out.next = NULL; - out.buf = b; - out.next = NULL; + r->headers_out.status = NGX_HTTP_OK; + r->headers_out.content_length_n = b->last - b->pos; - r->headers_out.status = NGX_HTTP_OK; - r->headers_out.content_length_n = b->last - b->pos; + b->last_buf = 1; + } - b->last_buf = 1; rc = ngx_http_send_header(r); if (rc == NGX_ERROR || rc > NGX_OK || r->header_only) { @@ -1304,6 +1628,7 @@ ngx_http_uploadprogress_merge_loc_conf(ngx_conf_t * cf, void *parent, void *chil } ngx_conf_merge_str_value(conf->header, prev->header, "X-Progress-ID"); + ngx_conf_merge_str_value(conf->header_mul, prev->header_mul, "X-ProgressMultiple-ID"); ngx_conf_merge_str_value(conf->jsonp_parameter, prev->jsonp_parameter, "callback"); return NGX_CONF_OK; @@ -1530,6 +1855,8 @@ ngx_http_upload_progress_template(ngx_conf_t * cf, ngx_command_t * cmd, void *co ngx_http_uploadprogress_state_map_t *m = ngx_http_uploadprogress_state_map; ngx_http_uploadprogress_template_t *t; + upcf->json_multiple = 0; + value = cf->args->elts; while(m->name.data != NULL) { @@ -1559,6 +1886,8 @@ ngx_http_upload_progress_java_output(ngx_conf_t * cf, ngx_command_t * cmd, void ngx_uint_t i; char* rc; + upcf->json_multiple = 0; + t = (ngx_http_uploadprogress_template_t*)upcf->templates.elts; for(i = 0;i < upcf->templates.nelts;i++) { @@ -1583,6 +1912,8 @@ ngx_http_upload_progress_json_output(ngx_conf_t * cf, ngx_command_t * cmd, void ngx_uint_t i; char* rc; + upcf->json_multiple = 0; + t = (ngx_http_uploadprogress_template_t*)upcf->templates.elts; for(i = 0;i < upcf->templates.nelts;i++) { @@ -1607,6 +1938,8 @@ ngx_http_upload_progress_jsonp_output(ngx_conf_t * cf, ngx_command_t * cmd, void ngx_uint_t i; char* rc; + upcf->json_multiple = 0; + t = (ngx_http_uploadprogress_template_t*)upcf->templates.elts; for(i = 0;i < upcf->templates.nelts;i++) { @@ -1623,6 +1956,32 @@ ngx_http_upload_progress_jsonp_output(ngx_conf_t * cf, ngx_command_t * cmd, void return NGX_CONF_OK; } +static char* +ngx_http_upload_progress_json_multiple_output(ngx_conf_t * cf, ngx_command_t * cmd, void *conf) +{ + ngx_http_uploadprogress_conf_t *upcf = conf; + ngx_http_uploadprogress_template_t *t; + ngx_uint_t i; + char* rc; + + upcf->json_multiple = 1; + + t = (ngx_http_uploadprogress_template_t*)upcf->templates.elts; + + for(i = 0;i < upcf->templates.nelts;i++) { + rc = ngx_http_upload_progress_set_template(cf, t + i, ngx_http_uploadprogress_json_multiple_defaults + i); + + if(rc != NGX_CONF_OK) { + return rc; + } + } + + upcf->content_type.data = (u_char*)"application/json"; + upcf->content_type.len = sizeof("application/json") - 1; + + return NGX_CONF_OK; +} + static ngx_int_t ngx_http_uploadprogress_received_variable(ngx_http_request_t *r, ngx_http_variable_value_t *v, uintptr_t data) { @@ -1699,6 +2058,30 @@ ngx_http_uploadprogress_status_variable(ngx_http_request_t *r, return NGX_OK; } +static ngx_int_t +ngx_http_uploadprogress_id_variable(ngx_http_request_t *r, + ngx_http_variable_value_t *v, uintptr_t data) +{ + ngx_http_uploadprogress_node_t *up; + u_char *p; + + up = ngx_http_get_module_ctx(r, ngx_http_uploadprogress_module); + + p = ngx_palloc(r->pool, up->len); + if (p == NULL) { + return NGX_ERROR; + } + + v->len = up->len; + v->data = p; + ngx_memcpy(v->data, up->data, up->len); + v->valid = 1; + v->no_cacheable = 0; + v->not_found = 0; + + return NGX_OK; +} + static ngx_int_t ngx_http_uploadprogress_callback_variable(ngx_http_request_t *r, ngx_http_variable_value_t *v, uintptr_t data) From 8423be4018083dff4d285cb5970c0da63a9feea6 Mon Sep 17 00:00:00 2001 From: devgs Date: Mon, 18 Jun 2012 17:23:07 +0300 Subject: [PATCH 2/6] removed extra header --- ngx_http_uploadprogress_module.c | 1 - 1 file changed, 1 deletion(-) diff --git a/ngx_http_uploadprogress_module.c b/ngx_http_uploadprogress_module.c index 7cf6d43..188fb06 100644 --- a/ngx_http_uploadprogress_module.c +++ b/ngx_http_uploadprogress_module.c @@ -8,7 +8,6 @@ #include #include -#include #define TIMER_FREQUENCY 15 * 1000 From 640830dab13fe930eeb846236a878e5aa33b3043 Mon Sep 17 00:00:00 2001 From: devgs Date: Tue, 25 Sep 2012 11:48:47 +0300 Subject: [PATCH 3/6] fix for malformed id strings --- ngx_http_uploadprogress_module.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ngx_http_uploadprogress_module.c b/ngx_http_uploadprogress_module.c index 188fb06..b89ea76 100644 --- a/ngx_http_uploadprogress_module.c +++ b/ngx_http_uploadprogress_module.c @@ -916,8 +916,8 @@ ngx_http_reportuploads_handler(ngx_http_request_t * r) r->headers_out.content_length_n += b->last - b->pos; p1 = p2 + 1; - offs += len + 1; } + offs += len + 1; } ngx_free(id); if(!p_chain_end) // Malformed id From 0c7662d166d1daf38cd1094813be6a3ea86022c1 Mon Sep 17 00:00:00 2001 From: devgs Date: Thu, 31 Jan 2013 12:36:17 +0200 Subject: [PATCH 4/6] Ability to notify remote server about upload progress via UDP messages --- ngx_http_uploadprogress_module.c | 78 +++++++++++++++++++++++++++++++- 1 file changed, 76 insertions(+), 2 deletions(-) diff --git a/ngx_http_uploadprogress_module.c b/ngx_http_uploadprogress_module.c index b89ea76..337c0a8 100644 --- a/ngx_http_uploadprogress_module.c +++ b/ngx_http_uploadprogress_module.c @@ -32,6 +32,7 @@ struct ngx_http_uploadprogress_node_s { off_t rest; off_t length; ngx_uint_t done; + ngx_uint_t sequence; time_t timeout; struct ngx_http_uploadprogress_node_s *prev; struct ngx_http_uploadprogress_node_s *next; @@ -68,6 +69,9 @@ typedef struct { ngx_str_t header; ngx_str_t header_mul; + ngx_addr_t progress_server; + int udp_socket; + ngx_str_t jsonp_parameter; ngx_int_t json_multiple:1; } ngx_http_uploadprogress_conf_t; @@ -104,6 +108,7 @@ static char* ngx_http_upload_progress_java_output(ngx_conf_t * cf, ngx_command_t static char* ngx_http_upload_progress_json_output(ngx_conf_t * cf, ngx_command_t * cmd, void *conf); static char* ngx_http_upload_progress_jsonp_output(ngx_conf_t * cf, ngx_command_t * cmd, void *conf); static char* ngx_http_upload_progress_json_multiple_output(ngx_conf_t * cf, ngx_command_t * cmd, void *conf); +static char* ngx_http_upload_progress_jsonp_multiple_output(ngx_conf_t * cf, ngx_command_t * cmd, void *conf); static void ngx_clean_old_connections(ngx_event_t * ev); static ngx_int_t ngx_http_uploadprogress_content_handler(ngx_http_request_t *r); @@ -119,7 +124,7 @@ static ngx_command_t ngx_http_uploadprogress_commands[] = { NULL}, {ngx_string("track_uploads"), - NGX_HTTP_MAIN_CONF | NGX_HTTP_SRV_CONF | NGX_HTTP_LOC_CONF | NGX_CONF_TAKE2, + NGX_HTTP_MAIN_CONF | NGX_HTTP_SRV_CONF | NGX_HTTP_LOC_CONF | NGX_CONF_TAKE2 | NGX_CONF_TAKE3, ngx_http_track_uploads, NGX_HTTP_LOC_CONF_OFFSET, 0, @@ -174,6 +179,13 @@ static ngx_command_t ngx_http_uploadprogress_commands[] = { 0, NULL}, + {ngx_string("upload_progress_jsonp_multiple_output"), + NGX_HTTP_MAIN_CONF | NGX_HTTP_SRV_CONF | NGX_HTTP_LOC_CONF | NGX_CONF_NOARGS, + ngx_http_upload_progress_jsonp_multiple_output, + NGX_HTTP_LOC_CONF_OFFSET, + 0, + NULL}, + {ngx_string("upload_progress_header"), NGX_HTTP_MAIN_CONF | NGX_HTTP_SRV_CONF | NGX_HTTP_LOC_CONF | NGX_CONF_TAKE1, ngx_conf_set_str_slot, @@ -293,6 +305,13 @@ static ngx_str_t ngx_http_uploadprogress_json_multiple_defaults[] = { ngx_string("{ \"id\" : $uploadprogress_id, \"state\" : \"uploading\", \"received\" : $uploadprogress_received, \"size\" : $uploadprogress_length }") }; +static ngx_str_t ngx_http_uploadprogress_jsonp_multiple_defaults[] = { + ngx_string("$uploadprogress_callback({ \"id\" : $uploadprogress_id, \"state\" : \"starting\" });\r\n"), + ngx_string("$uploadprogress_callback({ \"id\" : $uploadprogress_id, \"state\" : \"error\", \"status\" : $uploadprogress_status });\r\n"), + ngx_string("$uploadprogress_callback({ \"id\" : $uploadprogress_id, \"state\" : \"done\" });\r\n"), + ngx_string("$uploadprogress_callback({ \"id\" : $uploadprogress_id, \"state\" : \"uploading\", \"received\" : $uploadprogress_received, \"size\" : $uploadprogress_length });\r\n") +}; + static ngx_array_t ngx_http_uploadprogress_global_templates; @@ -644,6 +663,15 @@ static void ngx_http_uploadprogress_event_handler(ngx_http_request_t *r) up->rest = r->request_body->rest; if(up->length == 0) up->length = r->headers_in.content_length_n; + if(upcf->udp_socket != -1 && upcf->progress_server.socklen != 0) + { + u_char datagram_buf[1024]; + u_char * end; + + end = ngx_snprintf(datagram_buf, sizeof(datagram_buf), "{\"id\" : \"%V\", \"sequence\", \"size\" : %u, \"uploaded\" : %u}", id, up->sequence, up->length, up->rest); + sendto(upcf->udp_socket, datagram_buf, end - datagram_buf, 0, (struct sockaddr*)upcf->progress_server.sockaddr, upcf->progress_server.socklen); + ++up->sequence; + } ngx_log_debug3(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0, "upload-progress: read_event_handler storing rest %uO/%uO for %V", up->rest, up->length, id); } else { @@ -1155,6 +1183,7 @@ ngx_http_uploadprogress_handler(ngx_http_request_t * r) up->rest = 0; up->length = 0; up->timeout = 0; + up->sequence = 0; up->next = ctx->list_head.next; up->next->prev = up; @@ -1495,6 +1524,7 @@ ngx_http_uploadprogress_errortracker(ngx_http_request_t * r) up->rest = 0; up->length = 0; up->timeout = 0; + up->sequence = 0; ngx_memcpy(up->data, id->data, id->len); @@ -1746,6 +1776,7 @@ ngx_http_track_uploads(ngx_conf_t * cf, ngx_command_t * cmd, void *conf) ngx_http_core_loc_conf_t *clcf; ngx_http_uploadprogress_conf_t *lzcf = conf; ngx_str_t *value; + ngx_url_t url; ngx_log_debug0(NGX_LOG_DEBUG_HTTP, cf->log, 0, "ngx_track_uploads in"); @@ -1770,10 +1801,27 @@ ngx_http_track_uploads(ngx_conf_t * cf, ngx_command_t * cmd, void *conf) lzcf->timeout = ngx_parse_time(&value[2], 1); if (lzcf->timeout == NGX_ERROR) { ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, - "track_uploads \"%V\" timeout value invalid", &value[1]); + "track_uploads \"%V\" timeout value invalid", &value[2]); return NGX_CONF_ERROR; } + if(cf->args->nelts > 3) + { + ngx_memzero(&url, sizeof(ngx_url_t)); + url.url = value[3]; + url.default_port = 80; + url.no_resolve = 0; + + if(ngx_parse_url(cf->pool, &url) != NGX_OK) + { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "Invalid graphite server %V: %s", &url.host, url.err); + return NGX_CONF_ERROR; + } + lzcf->progress_server = url.addrs[0]; + if(lzcf->udp_socket == -1) + lzcf->udp_socket = ngx_socket(PF_INET, SOCK_DGRAM, 0); + } + clcf = ngx_http_conf_get_module_loc_conf(cf, ngx_http_core_module); lzcf->handler = clcf->handler; if ( lzcf->handler == NULL ) @@ -1981,6 +2029,32 @@ ngx_http_upload_progress_json_multiple_output(ngx_conf_t * cf, ngx_command_t * c return NGX_CONF_OK; } +static char* +ngx_http_upload_progress_jsonp_multiple_output(ngx_conf_t * cf, ngx_command_t * cmd, void *conf) +{ + ngx_http_uploadprogress_conf_t *upcf = conf; + ngx_http_uploadprogress_template_t *t; + ngx_uint_t i; + char* rc; + + upcf->json_multiple = 1; + + t = (ngx_http_uploadprogress_template_t*)upcf->templates.elts; + + for(i = 0;i < upcf->templates.nelts;i++) { + rc = ngx_http_upload_progress_set_template(cf, t + i, ngx_http_uploadprogress_jsonp_multiple_defaults + i); + + if(rc != NGX_CONF_OK) { + return rc; + } + } + + upcf->content_type.data = (u_char*)"application/json"; + upcf->content_type.len = sizeof("application/json") - 1; + + return NGX_CONF_OK; +} + static ngx_int_t ngx_http_uploadprogress_received_variable(ngx_http_request_t *r, ngx_http_variable_value_t *v, uintptr_t data) { From ac49788929370cdc279689042671b4554db15b58 Mon Sep 17 00:00:00 2001 From: devgs Date: Thu, 31 Jan 2013 14:37:19 +0200 Subject: [PATCH 5/6] socket initialization fix --- ngx_http_uploadprogress_module.c | 1 + 1 file changed, 1 insertion(+) diff --git a/ngx_http_uploadprogress_module.c b/ngx_http_uploadprogress_module.c index 337c0a8..ec12611 100644 --- a/ngx_http_uploadprogress_module.c +++ b/ngx_http_uploadprogress_module.c @@ -1616,6 +1616,7 @@ ngx_http_uploadprogress_create_loc_conf(ngx_conf_t * cf) elt->values = NULL; elt->lengths = NULL; } + conf->udp_socket = -1; return conf; } From 070e0368a0a472842e5d0a10cf9833ef96f6211f Mon Sep 17 00:00:00 2001 From: devgs Date: Thu, 21 Feb 2013 14:57:38 +0200 Subject: [PATCH 6/6] should be ready now --- ngx_http_uploadprogress_module.c | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/ngx_http_uploadprogress_module.c b/ngx_http_uploadprogress_module.c index ec12611..68af33b 100644 --- a/ngx_http_uploadprogress_module.c +++ b/ngx_http_uploadprogress_module.c @@ -33,6 +33,7 @@ struct ngx_http_uploadprogress_node_s { off_t length; ngx_uint_t done; ngx_uint_t sequence; + ngx_uint_t sent_portion; time_t timeout; struct ngx_http_uploadprogress_node_s *prev; struct ngx_http_uploadprogress_node_s *next; @@ -667,10 +668,22 @@ static void ngx_http_uploadprogress_event_handler(ngx_http_request_t *r) { u_char datagram_buf[1024]; u_char * end; - - end = ngx_snprintf(datagram_buf, sizeof(datagram_buf), "{\"id\" : \"%V\", \"sequence\", \"size\" : %u, \"uploaded\" : %u}", id, up->sequence, up->length, up->rest); - sendto(upcf->udp_socket, datagram_buf, end - datagram_buf, 0, (struct sockaddr*)upcf->progress_server.sockaddr, upcf->progress_server.socklen); - ++up->sequence; + off_t uploaded; + ngx_uint_t portion; + + uploaded = up->length - up->rest; + if(up->length) + portion = 100 * uploaded / up->length; + else + portion = 100; + if(portion > up->sent_portion) + { + end = ngx_snprintf(datagram_buf, sizeof(datagram_buf), "{\"id\" : \"%V\", \"sequence\" : %d, \"size\" : %uO, \"uploaded\" : %uO }", + id, up->sequence, up->length, uploaded); + sendto(upcf->udp_socket, datagram_buf, end - datagram_buf, 0, (struct sockaddr*)upcf->progress_server.sockaddr, upcf->progress_server.socklen); + up->sent_portion = portion; + ++up->sequence; + } } ngx_log_debug3(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0, "upload-progress: read_event_handler storing rest %uO/%uO for %V", up->rest, up->length, id); @@ -1184,6 +1197,7 @@ ngx_http_uploadprogress_handler(ngx_http_request_t * r) up->length = 0; up->timeout = 0; up->sequence = 0; + up->sent_portion = 0; up->next = ctx->list_head.next; up->next->prev = up; @@ -1525,6 +1539,7 @@ ngx_http_uploadprogress_errortracker(ngx_http_request_t * r) up->length = 0; up->timeout = 0; up->sequence = 0; + up->sent_portion = 0; ngx_memcpy(up->data, id->data, id->len);