1
2
3
4 """
5 This file is part of the web2py Web Framework
6 Copyrighted by Massimo Di Pierro <mdipierro@cs.depaul.edu>
7 License: LGPLv3 (http://www.gnu.org/licenses/lgpl.html)
8
9 Contains:
10
11 - wsgibase: the gluon wsgi application
12
13 """
14
15 import gc
16 import cgi
17 import cStringIO
18 import Cookie
19 import os
20 import re
21 import copy
22 import sys
23 import time
24 import thread
25 import datetime
26 import signal
27 import socket
28 import tempfile
29 import random
30 import string
31 import platform
32 from fileutils import abspath, write_file, parse_version
33 from settings import global_settings
34 from admin import add_path_first, create_missing_folders, create_missing_app_folders
35 from globals import current
36
37 from custom_import import custom_import_install
38 from contrib.simplejson import dumps
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56 if not hasattr(os, 'mkdir'):
57 global_settings.db_sessions = True
58 if global_settings.db_sessions is not True:
59 global_settings.db_sessions = set()
60 global_settings.gluon_parent = os.environ.get('web2py_path', os.getcwd())
61 global_settings.applications_parent = global_settings.gluon_parent
62 web2py_path = global_settings.applications_parent
63 global_settings.app_folders = set()
64 global_settings.debugging = False
65
66 custom_import_install(web2py_path)
67
68 create_missing_folders()
69
70
71 import logging
72 import logging.config
73 logpath = abspath("logging.conf")
74 if os.path.exists(logpath):
75 logging.config.fileConfig(abspath("logging.conf"))
76 else:
77 logging.basicConfig()
78 logger = logging.getLogger("web2py")
79
80 from restricted import RestrictedError
81 from http import HTTP, redirect
82 from globals import Request, Response, Session
83 from compileapp import build_environment, run_models_in, \
84 run_controller_in, run_view_in
85 from fileutils import copystream
86 from contenttype import contenttype
87 from dal import BaseAdapter
88 from settings import global_settings
89 from validators import CRYPT
90 from cache import Cache
91 from html import URL as Url
92 import newcron
93 import rewrite
94
95 __all__ = ['wsgibase', 'save_password', 'appfactory', 'HttpServer']
96
97 requests = 0
98
99
100
101
102
103 regex_client = re.compile('[\w\-:]+(\.[\w\-]+)*\.?')
104
105 version_info = open(abspath('VERSION', gluon=True), 'r')
106 web2py_version = parse_version(version_info.read().strip())
107 version_info.close()
108 global_settings.web2py_version = web2py_version
109
110 try:
111 import rocket
112 except:
113 if not global_settings.web2py_runtime_gae:
114 logger.warn('unable to import Rocket')
115
116 rewrite.load()
117
119 """
120 guess the client address from the environment variables
121
122 first tries 'http_x_forwarded_for', secondly 'remote_addr'
123 if all fails assume '127.0.0.1' (running locally)
124 """
125 g = regex_client.search(env.get('http_x_forwarded_for', ''))
126 if g:
127 return g.group()
128 g = regex_client.search(env.get('remote_addr', ''))
129 if g:
130 return g.group()
131 return '127.0.0.1'
132
134 """
135 copies request.env.wsgi_input into request.body
136 and stores progress upload status in cache.ram
137 X-Progress-ID:length and X-Progress-ID:uploaded
138 """
139 if not request.env.content_length:
140 return cStringIO.StringIO()
141 source = request.env.wsgi_input
142 size = int(request.env.content_length)
143 dest = tempfile.TemporaryFile()
144 if not 'X-Progress-ID' in request.vars:
145 copystream(source, dest, size, chunk_size)
146 return dest
147 cache_key = 'X-Progress-ID:'+request.vars['X-Progress-ID']
148 cache = Cache(request)
149 cache.ram(cache_key+':length', lambda: size, 0)
150 cache.ram(cache_key+':uploaded', lambda: 0, 0)
151 while size > 0:
152 if size < chunk_size:
153 data = source.read(size)
154 cache.ram.increment(cache_key+':uploaded', size)
155 else:
156 data = source.read(chunk_size)
157 cache.ram.increment(cache_key+':uploaded', chunk_size)
158 length = len(data)
159 if length > size:
160 (data, length) = (data[:size], size)
161 size -= length
162 if length == 0:
163 break
164 dest.write(data)
165 if length < chunk_size:
166 break
167 dest.seek(0)
168 cache.ram(cache_key+':length', None)
169 cache.ram(cache_key+':uploaded', None)
170 return dest
171
172
174 """
175 this function is used to generate a dynamic page.
176 It first runs all models, then runs the function in the controller,
177 and then tries to render the output using a view/template.
178 this function must run from the [application] folder.
179 A typical example would be the call to the url
180 /[application]/[controller]/[function] that would result in a call
181 to [function]() in applications/[application]/[controller].py
182 rendered by applications/[application]/views/[controller]/[function].html
183 """
184
185
186
187
188
189 environment = build_environment(request, response, session)
190
191
192
193 response.view = '%s/%s.%s' % (request.controller,
194 request.function,
195 request.extension)
196
197
198
199
200
201
202 run_models_in(environment)
203 response._view_environment = copy.copy(environment)
204 page = run_controller_in(request.controller, request.function, environment)
205 if isinstance(page, dict):
206 response._vars = page
207 for key in page:
208 response._view_environment[key] = page[key]
209 run_view_in(response._view_environment)
210 page = response.body.getvalue()
211
212 global requests
213 requests = ('requests' in globals()) and (requests+1) % 100 or 0
214 if not requests: gc.collect()
215
216 raise HTTP(response.status, page, **response.headers)
217
218
220 """
221 in controller you can use::
222
223 - request.wsgi.environ
224 - request.wsgi.start_response
225
226 to call third party WSGI applications
227 """
228 response.status = str(status).split(' ',1)[0]
229 response.headers = dict(headers)
230 return lambda *args, **kargs: response.write(escape=False,*args,**kargs)
231
232
234 """
235 In you controller use::
236
237 @request.wsgi.middleware(middleware1, middleware2, ...)
238
239 to decorate actions with WSGI middleware. actions must return strings.
240 uses a simulated environment so it may have weird behavior in some cases
241 """
242 def middleware(f):
243 def app(environ, start_response):
244 data = f()
245 start_response(response.status,response.headers.items())
246 if isinstance(data,list):
247 return data
248 return [data]
249 for item in middleware_apps:
250 app=item(app)
251 def caller(app):
252 return app(request.wsgi.environ,request.wsgi.start_response)
253 return lambda caller=caller, app=app: caller(app)
254 return middleware
255
257 new_environ = copy.copy(environ)
258 new_environ['wsgi.input'] = request.body
259 new_environ['wsgi.version'] = 1
260 return new_environ
261
262 -def parse_get_post_vars(request, environ):
263
264
265 dget = cgi.parse_qsl(request.env.query_string or '', keep_blank_values=1)
266 for (key, value) in dget:
267 if key in request.get_vars:
268 if isinstance(request.get_vars[key], list):
269 request.get_vars[key] += [value]
270 else:
271 request.get_vars[key] = [request.get_vars[key]] + [value]
272 else:
273 request.get_vars[key] = value
274 request.vars[key] = request.get_vars[key]
275
276
277 request.body = copystream_progress(request)
278 if (request.body and request.env.request_method in ('POST', 'PUT', 'BOTH')):
279 dpost = cgi.FieldStorage(fp=request.body,environ=environ,keep_blank_values=1)
280
281 is_multipart = dpost.type[:10] == 'multipart/'
282 request.body.seek(0)
283 isle25 = sys.version_info[1] <= 5
284
285 def listify(a):
286 return (not isinstance(a,list) and [a]) or a
287 try:
288 keys = sorted(dpost)
289 except TypeError:
290 keys = []
291 for key in keys:
292 dpk = dpost[key]
293
294 if isinstance(dpk, list):
295 if not dpk[0].filename:
296 value = [x.value for x in dpk]
297 else:
298 value = [x for x in dpk]
299 elif not dpk.filename:
300 value = dpk.value
301 else:
302 value = dpk
303 pvalue = listify(value)
304 if key in request.vars:
305 gvalue = listify(request.vars[key])
306 if isle25:
307 value = pvalue + gvalue
308 elif is_multipart:
309 pvalue = pvalue[len(gvalue):]
310 else:
311 pvalue = pvalue[:-len(gvalue)]
312 request.vars[key] = value
313 if len(pvalue):
314 request.post_vars[key] = (len(pvalue)>1 and pvalue) or pvalue[0]
315
316
318 """
319 this is the gluon wsgi application. the first function called when a page
320 is requested (static or dynamic). it can be called by paste.httpserver
321 or by apache mod_wsgi.
322
323 - fills request with info
324 - the environment variables, replacing '.' with '_'
325 - adds web2py path and version info
326 - compensates for fcgi missing path_info and query_string
327 - validates the path in url
328
329 The url path must be either:
330
331 1. for static pages:
332
333 - /<application>/static/<file>
334
335 2. for dynamic pages:
336
337 - /<application>[/<controller>[/<function>[/<sub>]]][.<extension>]
338 - (sub may go several levels deep, currently 3 levels are supported:
339 sub1/sub2/sub3)
340
341 The naming conventions are:
342
343 - application, controller, function and extension may only contain
344 [a-zA-Z0-9_]
345 - file and sub may also contain '-', '=', '.' and '/'
346 """
347
348 current.__dict__.clear()
349 request = Request()
350 response = Response()
351 session = Session()
352 request.env.web2py_path = global_settings.applications_parent
353 request.env.web2py_version = web2py_version
354 request.env.update(global_settings)
355 static_file = False
356 try:
357 try:
358 try:
359
360
361
362
363
364
365
366
367
368 if not environ.get('PATH_INFO',None) and \
369 environ.get('REQUEST_URI',None):
370
371 items = environ['REQUEST_URI'].split('?')
372 environ['PATH_INFO'] = items[0]
373 if len(items) > 1:
374 environ['QUERY_STRING'] = items[1]
375 else:
376 environ['QUERY_STRING'] = ''
377 if not environ.get('HTTP_HOST',None):
378 environ['HTTP_HOST'] = '%s:%s' % (environ.get('SERVER_NAME'),
379 environ.get('SERVER_PORT'))
380
381 (static_file, environ) = rewrite.url_in(request, environ)
382 if static_file:
383 if request.env.get('query_string', '')[:10] == 'attachment':
384 response.headers['Content-Disposition'] = 'attachment'
385 response.stream(static_file, request=request)
386
387
388
389
390
391 http_host = request.env.http_host.split(':',1)[0]
392
393 local_hosts = [http_host,'::1','127.0.0.1','::ffff:127.0.0.1']
394 if not global_settings.web2py_runtime_gae:
395 local_hosts += [socket.gethostname(),
396 socket.gethostbyname(http_host)]
397 request.client = get_client(request.env)
398 request.folder = abspath('applications',
399 request.application) + os.sep
400 x_req_with = str(request.env.http_x_requested_with).lower()
401 request.ajax = x_req_with == 'xmlhttprequest'
402 request.cid = request.env.http_web2py_component_element
403 request.is_local = request.env.remote_addr in local_hosts
404 request.is_https = request.env.wsgi_url_scheme \
405 in ['https', 'HTTPS'] or request.env.https == 'on'
406
407
408
409
410
411 response.uuid = request.compute_uuid()
412
413
414
415
416
417 if not os.path.exists(request.folder):
418 if request.application == rewrite.thread.routes.default_application and request.application != 'welcome':
419 request.application = 'welcome'
420 redirect(Url(r=request))
421 elif rewrite.thread.routes.error_handler:
422 _handler = rewrite.thread.routes.error_handler
423 redirect(Url(_handler['application'],
424 _handler['controller'],
425 _handler['function'],
426 args=request.application))
427 else:
428 raise HTTP(404, rewrite.thread.routes.error_message \
429 % 'invalid request',
430 web2py_error='invalid application')
431 request.url = Url(r=request, args=request.args,
432 extension=request.raw_extension)
433
434
435
436
437
438 create_missing_app_folders(request)
439
440
441
442
443
444 parse_get_post_vars(request, environ)
445
446
447
448
449
450 request.wsgi.environ = environ_aux(environ,request)
451 request.wsgi.start_response = \
452 lambda status='200', headers=[], \
453 exec_info=None, response=response: \
454 start_response_aux(status, headers, exec_info, response)
455 request.wsgi.middleware = \
456 lambda *a: middleware_aux(request,response,*a)
457
458
459
460
461
462 if request.env.http_cookie:
463 try:
464 request.cookies.load(request.env.http_cookie)
465 except Cookie.CookieError, e:
466 pass
467
468
469
470
471
472 session.connect(request, response)
473
474
475
476
477
478 response.headers['Content-Type'] = \
479 contenttype('.'+request.extension)
480 response.headers['Cache-Control'] = \
481 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0'
482 response.headers['Expires'] = \
483 time.strftime('%a, %d %b %Y %H:%M:%S GMT', time.gmtime())
484 response.headers['Pragma'] = 'no-cache'
485
486
487
488
489
490 serve_controller(request, response, session)
491
492 except HTTP, http_response:
493 if static_file:
494 return http_response.to(responder)
495
496 if request.body:
497 request.body.close()
498
499
500
501
502 session._try_store_in_db(request, response)
503
504
505
506
507
508 if response._custom_commit:
509 response._custom_commit()
510 else:
511 BaseAdapter.close_all_instances('commit')
512
513
514
515
516
517
518 session._try_store_on_disk(request, response)
519
520
521
522
523
524 if request.cid:
525
526 if response.flash and not 'web2py-component-flash' in http_response.headers:
527 http_response.headers['web2py-component-flash'] = \
528 str(response.flash).replace('\n','')
529 if response.js and not 'web2py-component-command' in http_response.headers:
530 http_response.headers['web2py-component-command'] = \
531 response.js.replace('\n','')
532 if session._forget and \
533 response.session_id_name in response.cookies:
534 del response.cookies[response.session_id_name]
535 elif session._secure:
536 response.cookies[response.session_id_name]['secure'] = True
537 if len(response.cookies)>0:
538 http_response.headers['Set-Cookie'] = \
539 [str(cookie)[11:] for cookie in response.cookies.values()]
540 ticket=None
541
542 except RestrictedError, e:
543
544 if request.body:
545 request.body.close()
546
547
548
549
550
551 ticket = e.log(request) or 'unknown'
552 if response._custom_rollback:
553 response._custom_rollback()
554 else:
555 BaseAdapter.close_all_instances('rollback')
556
557 http_response = \
558 HTTP(500, rewrite.thread.routes.error_message_ticket % \
559 dict(ticket=ticket),
560 web2py_error='ticket %s' % ticket)
561
562 except:
563
564 if request.body:
565 request.body.close()
566
567
568
569
570
571 try:
572 if response._custom_rollback:
573 response._custom_rollback()
574 else:
575 BaseAdapter.close_all_instances('rollback')
576 except:
577 pass
578 e = RestrictedError('Framework', '', '', locals())
579 ticket = e.log(request) or 'unrecoverable'
580 http_response = \
581 HTTP(500, rewrite.thread.routes.error_message_ticket \
582 % dict(ticket=ticket),
583 web2py_error='ticket %s' % ticket)
584
585 finally:
586 if response and hasattr(response, 'session_file') \
587 and response.session_file:
588 response.session_file.close()
589
590
591
592
593 session._unlock(response)
594 http_response, new_environ = rewrite.try_rewrite_on_error(
595 http_response, request, environ, ticket)
596 if not http_response:
597 return wsgibase(new_environ,responder)
598 if global_settings.web2py_crontype == 'soft':
599 newcron.softcron(global_settings.applications_parent).start()
600 return http_response.to(responder)
601
602
604 """
605 used by main() to save the password in the parameters_port.py file.
606 """
607
608 password_file = abspath('parameters_%i.py' % port)
609 if password == '<random>':
610
611 chars = string.letters + string.digits
612 password = ''.join([random.choice(chars) for i in range(8)])
613 cpassword = CRYPT()(password)[0]
614 print '******************* IMPORTANT!!! ************************'
615 print 'your admin password is "%s"' % password
616 print '*********************************************************'
617 elif password == '<recycle>':
618
619 if os.path.exists(password_file):
620 return
621 else:
622 password = ''
623 elif password.startswith('<pam_user:'):
624
625 cpassword = password[1:-1]
626 else:
627
628 cpassword = CRYPT()(password)[0]
629 fp = open(password_file, 'w')
630 if password:
631 fp.write('password="%s"\n' % cpassword)
632 else:
633 fp.write('password=None\n')
634 fp.close()
635
636
637 -def appfactory(wsgiapp=wsgibase,
638 logfilename='httpserver.log',
639 profilerfilename='profiler.log'):
640 """
641 generates a wsgi application that does logging and profiling and calls
642 wsgibase
643
644 .. function:: gluon.main.appfactory(
645 [wsgiapp=wsgibase
646 [, logfilename='httpserver.log'
647 [, profilerfilename='profiler.log']]])
648
649 """
650 if profilerfilename and os.path.exists(profilerfilename):
651 os.unlink(profilerfilename)
652 locker = thread.allocate_lock()
653
654 def app_with_logging(environ, responder):
655 """
656 a wsgi app that does logging and profiling and calls wsgibase
657 """
658 status_headers = []
659
660 def responder2(s, h):
661 """
662 wsgi responder app
663 """
664 status_headers.append(s)
665 status_headers.append(h)
666 return responder(s, h)
667
668 time_in = time.time()
669 ret = [0]
670 if not profilerfilename:
671 ret[0] = wsgiapp(environ, responder2)
672 else:
673 import cProfile
674 import pstats
675 logger.warn('profiler is on. this makes web2py slower and serial')
676
677 locker.acquire()
678 cProfile.runctx('ret[0] = wsgiapp(environ, responder2)',
679 globals(), locals(), profilerfilename+'.tmp')
680 stat = pstats.Stats(profilerfilename+'.tmp')
681 stat.stream = cStringIO.StringIO()
682 stat.strip_dirs().sort_stats("time").print_stats(80)
683 profile_out = stat.stream.getvalue()
684 profile_file = open(profilerfilename, 'a')
685 profile_file.write('%s\n%s\n%s\n%s\n\n' % \
686 ('='*60, environ['PATH_INFO'], '='*60, profile_out))
687 profile_file.close()
688 locker.release()
689 try:
690 line = '%s, %s, %s, %s, %s, %s, %f\n' % (
691 environ['REMOTE_ADDR'],
692 datetime.datetime.today().strftime('%Y-%m-%d %H:%M:%S'),
693 environ['REQUEST_METHOD'],
694 environ['PATH_INFO'].replace(',', '%2C'),
695 environ['SERVER_PROTOCOL'],
696 (status_headers[0])[:3],
697 time.time() - time_in,
698 )
699 if not logfilename:
700 sys.stdout.write(line)
701 elif isinstance(logfilename, str):
702 write_file(logfilename, line, 'a')
703 else:
704 logfilename.write(line)
705 except:
706 pass
707 return ret[0]
708
709 return app_with_logging
710
711
713 """
714 the web2py web server (Rocket)
715 """
716
717 - def __init__(
718 self,
719 ip='127.0.0.1',
720 port=8000,
721 password='',
722 pid_filename='httpserver.pid',
723 log_filename='httpserver.log',
724 profiler_filename=None,
725 ssl_certificate=None,
726 ssl_private_key=None,
727 ssl_ca_certificate=None,
728 min_threads=None,
729 max_threads=None,
730 server_name=None,
731 request_queue_size=5,
732 timeout=10,
733 shutdown_timeout=None,
734 path=None,
735 interfaces=None
736 ):
737 """
738 starts the web server.
739 """
740
741 if interfaces:
742
743
744 import types
745 if isinstance(interfaces,types.ListType):
746 for i in interfaces:
747 if not isinstance(i,types.TupleType):
748 raise "Wrong format for rocket interfaces parameter - see http://packages.python.org/rocket/"
749 else:
750 raise "Wrong format for rocket interfaces parameter - see http://packages.python.org/rocket/"
751
752 if path:
753
754
755 global web2py_path
756 path = os.path.normpath(path)
757 web2py_path = path
758 global_settings.applications_parent = path
759 os.chdir(path)
760 [add_path_first(p) for p in (path, abspath('site-packages'), "")]
761
762 save_password(password, port)
763 self.pid_filename = pid_filename
764 if not server_name:
765 server_name = socket.gethostname()
766 logger.info('starting web server...')
767 rocket.SERVER_NAME = server_name
768 sock_list = [ip, port]
769 if not ssl_certificate or not ssl_private_key:
770 logger.info('SSL is off')
771 elif not rocket.ssl:
772 logger.warning('Python "ssl" module unavailable. SSL is OFF')
773 elif not os.path.exists(ssl_certificate):
774 logger.warning('unable to open SSL certificate. SSL is OFF')
775 elif not os.path.exists(ssl_private_key):
776 logger.warning('unable to open SSL private key. SSL is OFF')
777 else:
778 sock_list.extend([ssl_private_key, ssl_certificate])
779 if ssl_ca_certificate:
780 sock_list.append(ssl_ca_certificate)
781
782 logger.info('SSL is ON')
783 app_info = {'wsgi_app': appfactory(wsgibase,
784 log_filename,
785 profiler_filename) }
786
787 self.server = rocket.Rocket(interfaces or tuple(sock_list),
788 method='wsgi',
789 app_info=app_info,
790 min_threads=min_threads,
791 max_threads=max_threads,
792 queue_size=int(request_queue_size),
793 timeout=int(timeout),
794 handle_signals=False,
795 )
796
797
799 """
800 start the web server
801 """
802 try:
803 signal.signal(signal.SIGTERM, lambda a, b, s=self: s.stop())
804 signal.signal(signal.SIGINT, lambda a, b, s=self: s.stop())
805 except:
806 pass
807 write_file(self.pid_filename, str(os.getpid()))
808 self.server.start()
809
810 - def stop(self, stoplogging=False):
811 """
812 stop cron and the web server
813 """
814 newcron.stopcron()
815 self.server.stop(stoplogging)
816 try:
817 os.unlink(self.pid_filename)
818 except:
819 pass
820