mirror of
https://github.com/NohamR/knowledge-kit.git
synced 2026-05-24 20:00:37 +00:00
docs: Python 脚本实现爬取疫情数据并用 Markdown 预览
This commit is contained in:
杭城小刘
@@ -1861,7 +1861,7 @@ iOS 网络现状是由4层组成的：最底层的 BSD Sockets、SecureTransport
|
||||
|
||||
|
||||
|
||||
#### 2.1 NSURLProtocol 监控 App 网络请求
|
||||
#### 2.1 NSURLProtocol 监控 App 网络请求<a name="network-2.1"></a>
|
||||
|
||||
NSURLProtocol 作为上层接口，使用较为简单，但 NSURLProtocol 属于 URL Loading System 体系中。应用协议的支持程度有限，支持 FTP、HTTP、HTTPS 等几个应用层协议，对于其他的协议则无法监控，存在一定的局限性。如果监控底层网络库 CFNetwork 则没有这个限制。
|
||||
|
||||
@@ -2341,9 +2341,9 @@ API_AVAILABLE(macosx(10.12), ios(10.0), watchos(3.0), tvos(10.0))
|
||||
|
||||
|
||||
|
||||
#### 2.2 骚操作篇
|
||||
#### 2.2 骚操作篇 <a name="network-2.2"></a>
|
||||
|
||||
在 2.1 分析到了 NSURLSessionTaskMetrics 由于兼容性问题，对于网络监控来说似乎不太完美，但是自后在搜资料的时候看到了一篇[文章](https://www.jianshu.com/p/1c34147030d1)。文章在分析 WebView 的网络监控的时候分析 Webkit 源码的时候发现了下面代码
|
||||
文章上面 [2.1 ](#network-2.1)分析到了 NSURLSessionTaskMetrics 由于兼容性问题，对于网络监控来说似乎不太完美，但是自后在搜资料的时候看到了一篇[文章](https://www.jianshu.com/p/1c34147030d1)。文章在分析 WebView 的网络监控的时候分析 Webkit 源码的时候发现了下面代码
|
||||
|
||||
```objective-c
|
||||
#if !HAVE(TIMINGDATAOPTIONS)
|
||||
@@ -2441,9 +2441,9 @@ NSURLSession 在 iOS9 之前使用 `_setCollectsTimingData:` 就可以使用 Tim
|
||||
|
||||
在 iOS 中 AOP 的实现是基于 Runtime 机制，目前由3种方式：Method Swizzling、NSProxy、FishHook（主要用用于 hook c 代码）。
|
||||
|
||||
2.1 讨论了满足大多数的需求的场景，NSURLProtocol 监控了 NSURLConnection、NSURLSession 的网络请求，自身代理后可以发起网络请求并得到诸如请求开始时间、请求结束时间、header 信息等，但是无法得到非常详细的网络性能数据，比如 DNS 开始解析时间、DNS 解析用了多久、reponse 开始返回的时间、返回了多久等。 iOS10 之后 NSURLSessionTaskDelegate 增加了一个代理方法 `- (void)URLSession:(NSURLSession *)session task:(NSURLSessionTask *)task didFinishCollectingMetrics:(NSURLSessionTaskMetrics *)metrics API_AVAILABLE(macosx(10.12), ios(10.0), watchos(3.0), tvos(10.0));`，可以获取到精确的各项网络数据。但是具有兼容性。2.2 讨论了一个 Webkit 源码中得到的信息，通过私有方法 `_setCollectsTimingData:` 、`_timingData` 可以获取到 TimingData。
|
||||
文章上面 [2.1 ](#network-2.1)讨论了满足大多数的需求的场景，NSURLProtocol 监控了 NSURLConnection、NSURLSession 的网络请求，自身代理后可以发起网络请求并得到诸如请求开始时间、请求结束时间、header 信息等，但是无法得到非常详细的网络性能数据，比如 DNS 开始解析时间、DNS 解析用了多久、reponse 开始返回的时间、返回了多久等。 iOS10 之后 NSURLSessionTaskDelegate 增加了一个代理方法 `- (void)URLSession:(NSURLSession *)session task:(NSURLSessionTask *)task didFinishCollectingMetrics:(NSURLSessionTaskMetrics *)metrics API_AVAILABLE(macosx(10.12), ios(10.0), watchos(3.0), tvos(10.0));`，可以获取到精确的各项网络数据。但是具有兼容性。文章上面 [2.2 ](#network-2.2)讨论了从 Webkit 源码中得到的信息，通过私有方法 `_setCollectsTimingData:` 、`_timingData` 可以获取到 TimingData。
|
||||
|
||||
但是如果需要监全部的网络请求就不能满足需求了，查阅资料后发现了阿里百川有 APM 的解决方案，对于网络监控需要做如下的 hook
|
||||
但是如果需要监全部的网络请求就不能满足需求了，查阅资料后发现了阿里百川有 APM 的解决方案，于是有了方案3，对于网络监控需要做如下的处理
|
||||
|
||||
|
||||
|
||||
@@ -2546,7 +2546,292 @@ void printResponseData (CFDataRef responseData) {
|
||||
}
|
||||
```
|
||||
|
||||
Method Swizzling 方法替换需要知道类名，我们在开发阶段使用 NSURLConnection、NSURLSession 都需要指定代理对象，且代理对象在该阶段没有办法确定，所以在此处的思路是在 NSURLConnection、NSURLSession 设置代理的时候替换掉代理对象，
|
||||
我们知道 NSURLSession、NSURLConnection、CFNetwork 的使用都需要调用一堆方法进行设置然后需要设置代理对象，实现代理方法。所以针对这种情况进行监控首先想到的是使用 runtime hook 掉方法层级。但是针对设置的代理对象的代理方法没办法 hook，因为不知道代理对象是哪个类。所以想办法可以 hook 设置代理对象这个步骤，将代理对象替换成我们设计好的某个类，然后让这个类去实现 NSURLConnection、NSURLSession、CFNetwork 相关的代理方法。然后在这些方法的内部都去调用一下原代理对象的方法实现。所以我们的需求得以满足，我们在相应的方法里面可以拿到监控数据，比如请求开始时间、结束时间、状态码、内容大小等。
|
||||
|
||||
具体步骤及其关键代码如下，以 NSURLConnection 举例
|
||||
|
||||
- 因为要 Hook 挺多地方，所以写一个 method swizzling 的工具类
|
||||
|
||||
```objective-c
|
||||
#import <Foundation/Foundation.h>
|
||||
|
||||
NS_ASSUME_NONNULL_BEGIN
|
||||
|
||||
@interface NSObject (hook)
|
||||
|
||||
/**
|
||||
hook对象方法
|
||||
|
||||
@param originalSelector 需要hook的原始对象方法
|
||||
@param swizzledSelector 需要替换的对象方法
|
||||
*/
|
||||
+ (void)apm_swizzleMethod:(SEL)originalSelector swizzledSelector:(SEL)swizzledSelector;
|
||||
|
||||
/**
|
||||
hook类方法
|
||||
|
||||
@param originalSelector 需要hook的原始类方法
|
||||
@param swizzledSelector 需要替换的类方法
|
||||
*/
|
||||
+ (void)apm_swizzleClassMethod:(SEL)originalSelector swizzledSelector:(SEL)swizzledSelector;
|
||||
|
||||
@end
|
||||
|
||||
NS_ASSUME_NONNULL_END
|
||||
|
||||
+ (void)apm_swizzleMethod:(SEL)originalSelector swizzledSelector:(SEL)swizzledSelector
|
||||
{
|
||||
class_swizzleInstanceMethod(self, originalSelector, swizzledSelector);
|
||||
}
|
||||
|
||||
+ (void)apm_swizzleClassMethod:(SEL)originalSelector swizzledSelector:(SEL)swizzledSelector
|
||||
{
|
||||
//类方法实际上是储存在类对象的类(即元类)中，即类方法相当于元类的实例方法,所以只需要把元类传入，其他逻辑和交互实例方法一样。
|
||||
Class class2 = object_getClass(self);
|
||||
class_swizzleInstanceMethod(class2, originalSelector, swizzledSelector);
|
||||
}
|
||||
|
||||
void class_swizzleInstanceMethod(Class class, SEL originalSEL, SEL replacementSEL)
|
||||
{
|
||||
Method originMethod = class_getInstanceMethod(class, originalSEL);
|
||||
Method replaceMethod = class_getInstanceMethod(class, replacementSEL);
|
||||
|
||||
if(class_addMethod(class, originalSEL, method_getImplementation(replaceMethod),method_getTypeEncoding(replaceMethod)))
|
||||
{
|
||||
class_replaceMethod(class,replacementSEL, method_getImplementation(originMethod), method_getTypeEncoding(originMethod));
|
||||
}else {
|
||||
method_exchangeImplementations(originMethod, replaceMethod);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- 建立一个继承自 NSProxy 抽象类的类，实现相应方法。
|
||||
|
||||
```objective-c
|
||||
#import <Foundation/Foundation.h>
|
||||
|
||||
NS_ASSUME_NONNULL_BEGIN
|
||||
|
||||
// 为 NSURLConnection、NSURLSession、CFNetwork 代理设置代理转发
|
||||
@interface NetworkDelegateProxy : NSProxy
|
||||
|
||||
+ (instancetype)setProxyForObject:(id)originalTarget withNewDelegate:(id)newDelegate;
|
||||
|
||||
@end
|
||||
|
||||
NS_ASSUME_NONNULL_END
|
||||
|
||||
// .m
|
||||
@interface NetworkDelegateProxy () {
|
||||
id _originalTarget;
|
||||
id _NewDelegate;
|
||||
}
|
||||
|
||||
@end
|
||||
|
||||
|
||||
@implementation NetworkDelegateProxy
|
||||
|
||||
#pragma mark - life cycle
|
||||
|
||||
+ (instancetype)sharedInstance {
|
||||
static NetworkDelegateProxy *_sharedInstance = nil;
|
||||
|
||||
static dispatch_once_t onceToken;
|
||||
|
||||
dispatch_once(&onceToken, ^{
|
||||
_sharedInstance = [NetworkDelegateProxy alloc];
|
||||
});
|
||||
|
||||
return _sharedInstance;
|
||||
}
|
||||
|
||||
|
||||
#pragma mark - public Method
|
||||
|
||||
+ (instancetype)setProxyForObject:(id)originalTarget withNewDelegate:(id)newDelegate
|
||||
{
|
||||
NetworkDelegateProxy *instance = [NetworkDelegateProxy sharedInstance];
|
||||
instance->_originalTarget = originalTarget;
|
||||
instance->_NewDelegate = newDelegate;
|
||||
return instance;
|
||||
}
|
||||
|
||||
- (void)forwardInvocation:(NSInvocation *)invocation
|
||||
{
|
||||
if ([_originalTarget respondsToSelector:invocation.selector]) {
|
||||
[invocation invokeWithTarget:_originalTarget];
|
||||
[((NSURLSessionAndConnectionImplementor *)_NewDelegate) invoke:invocation];
|
||||
}
|
||||
}
|
||||
|
||||
- (nullable NSMethodSignature *)methodSignatureForSelector:(SEL)sel
|
||||
{
|
||||
return [_originalTarget methodSignatureForSelector:sel];
|
||||
}
|
||||
|
||||
@end
|
||||
```
|
||||
|
||||
- 创建一个对象，实现 NSURLConnection、NSURLSession、NSIuputStream 代理方法
|
||||
|
||||
```objective-c
|
||||
// NetworkImplementor.m
|
||||
|
||||
#pragma mark-NSURLConnectionDelegate
|
||||
- (void)connection:(NSURLConnection *)connection didFailWithError:(NSError *)error {
|
||||
NSLog(@"%s", __func__);
|
||||
}
|
||||
|
||||
- (nullable NSURLRequest *)connection:(NSURLConnection *)connection willSendRequest:(NSURLRequest *)request redirectResponse:(nullable NSURLResponse *)response {
|
||||
NSLog(@"%s", __func__);
|
||||
return request;
|
||||
}
|
||||
|
||||
#pragma mark-NSURLConnectionDataDelegate
|
||||
- (void)connection:(NSURLConnection *)connection didReceiveResponse:(NSURLResponse *)response {
|
||||
NSLog(@"%s", __func__);
|
||||
}
|
||||
|
||||
- (void)connection:(NSURLConnection *)connection didReceiveData:(NSData *)data {
|
||||
NSLog(@"%s", __func__);
|
||||
}
|
||||
|
||||
- (void)connection:(NSURLConnection *)connection didSendBodyData:(NSInteger)bytesWritten
|
||||
totalBytesWritten:(NSInteger)totalBytesWritten
|
||||
totalBytesExpectedToWrite:(NSInteger)totalBytesExpectedToWrite {
|
||||
NSLog(@"%s", __func__);
|
||||
}
|
||||
|
||||
- (void)connectionDidFinishLoading:(NSURLConnection *)connection {
|
||||
NSLog(@"%s", __func__);
|
||||
}
|
||||
|
||||
#pragma mark-NSURLConnectionDownloadDelegate
|
||||
- (void)connection:(NSURLConnection *)connection didWriteData:(long long)bytesWritten totalBytesWritten:(long long)totalBytesWritten expectedTotalBytes:(long long) expectedTotalBytes {
|
||||
NSLog(@"%s", __func__);
|
||||
}
|
||||
|
||||
- (void)connectionDidResumeDownloading:(NSURLConnection *)connection totalBytesWritten:(long long)totalBytesWritten expectedTotalBytes:(long long) expectedTotalBytes {
|
||||
NSLog(@"%s", __func__);
|
||||
}
|
||||
|
||||
- (void)connectionDidFinishDownloading:(NSURLConnection *)connection destinationURL:(NSURL *) destinationURL {
|
||||
NSLog(@"%s", __func__);
|
||||
}
|
||||
// 根据需求自己去写需要监控的数据项
|
||||
```
|
||||
|
||||
- 给 NSURLConnection 添加 Category，专门设置 hook 代理对象、hook NSURLConnection 对象方法
|
||||
|
||||
```objective-c
|
||||
// NSURLConnection+Monitor.m
|
||||
@implementation NSURLConnection (Monitor)
|
||||
|
||||
+ (void)load
|
||||
{
|
||||
static dispatch_once_t onceToken;
|
||||
dispatch_once(&onceToken, ^{
|
||||
@autoreleasepool {
|
||||
[[self class] apm_swizzleMethod:@selector(apm_initWithRequest:delegate:) swizzledSelector:@selector(initWithRequest: delegate:)];
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
- (_Nonnull instancetype)apm_initWithRequest:(NSURLRequest *)request delegate:(nullable id)delegate
|
||||
{
|
||||
/*
|
||||
1. 在设置 Delegate 的时候替换 delegate。
|
||||
2. 因为要在每个代理方法里面，监控数据，所以需要将代理方法都 hook 下
|
||||
3. 在原代理方法执行的时候，让新的代理对象里面，去执行方法的转发，
|
||||
*/
|
||||
NSString *traceId = @"traceId";
|
||||
NSMutableURLRequest *rq = [request mutableCopy];
|
||||
NSString *preTraceId = [request.allHTTPHeaderFields valueForKey:@"head_key_traceid"];
|
||||
if (preTraceId) {
|
||||
// 调用 hook 之前的初始化方法，返回 NSURLConnection
|
||||
return [self apm_initWithRequest:rq delegate:delegate];
|
||||
} else {
|
||||
[rq setValue:traceId forHTTPHeaderField:@"head_key_traceid"];
|
||||
|
||||
NSURLSessionAndConnectionImplementor *mockDelegate = [NSURLSessionAndConnectionImplementor new];
|
||||
[self registerDelegateMethod:@"connection:didFailWithError:" originalDelegate:delegate newDelegate:mockDelegate flag:"v@:@@"];
|
||||
|
||||
[self registerDelegateMethod:@"connection:didReceiveResponse:" originalDelegate:delegate newDelegate:mockDelegate flag:"v@:@@"];
|
||||
[self registerDelegateMethod:@"connection:didReceiveData:" originalDelegate:delegate newDelegate:mockDelegate flag:"v@:@@"];
|
||||
[self registerDelegateMethod:@"connection:didFailWithError:" originalDelegate:delegate newDelegate:mockDelegate flag:"v@:@@"];
|
||||
|
||||
[self registerDelegateMethod:@"connectionDidFinishLoading:" originalDelegate:delegate newDelegate:mockDelegate flag:"v@:@"];
|
||||
[self registerDelegateMethod:@"connection:willSendRequest:redirectResponse:" originalDelegate:delegate newDelegate:mockDelegate flag:"@@:@@"];
|
||||
delegate = [NetworkDelegateProxy setProxyForObject:delegate withNewDelegate:mockDelegate];
|
||||
|
||||
// 调用 hook 之前的初始化方法，返回 NSURLConnection
|
||||
return [self apm_initWithRequest:rq delegate:delegate];
|
||||
}
|
||||
}
|
||||
|
||||
- (void)registerDelegateMethod:(NSString *)methodName originalDelegate:(id<NSURLConnectionDelegate>)originalDelegate newDelegate:(NSURLSessionAndConnectionImplementor *)newDelegate flag:(const char *)flag
|
||||
{
|
||||
if ([originalDelegate respondsToSelector:NSSelectorFromString(methodName)]) {
|
||||
IMP originalMethodImp = class_getMethodImplementation([originalDelegate class], NSSelectorFromString(methodName));
|
||||
IMP newMethodImp = class_getMethodImplementation([newDelegate class], NSSelectorFromString(methodName));
|
||||
if (originalMethodImp != newMethodImp) {
|
||||
[newDelegate registerSelector: methodName];
|
||||
NSLog(@"");
|
||||
}
|
||||
} else {
|
||||
class_addMethod([originalDelegate class], NSSelectorFromString(methodName), class_getMethodImplementation([newDelegate class], NSSelectorFromString(methodName)), flag);
|
||||
}
|
||||
}
|
||||
|
||||
@end
|
||||
```
|
||||
|
||||
|
||||
|
||||
这样下来就是可以监控到网络信息了，然后将数据交给数据上报 SDK，按照下发的数据上报策略去上报数据。
|
||||
|
||||
顺道说一句，上面针对 NSURLConnection、NSURLSession、NSInputStream 代理对象的 hook 之后，利用 NSProxy 实现代理对象方法的转发，有另一种方法可以实现，那就是 **isa swizzling**。
|
||||
|
||||
- Method swizzling 原理
|
||||
|
||||
```objective-c
|
||||
struct old_method {
|
||||
SEL method_name;
|
||||
char *method_types;
|
||||
IMP method_imp;
|
||||
};
|
||||
```
|
||||
|
||||
|
||||
|
||||
method swizzling 改进版如下
|
||||
|
||||
```objective-c
|
||||
Method originalMethod = class_getInstanceMethod(aClass, aSEL);
|
||||
IMP originalIMP = method_getImplementation(originalMethod);
|
||||
char *cd = method_getTypeEncoding(originalMethod);
|
||||
IMP newIMP = imp_implementationWithBlock(^(id self) {
|
||||
void (*tmp)(id self, SEL _cmd) = originalIMP;
|
||||
tmp(self, aSEL);
|
||||
});
|
||||
class_replaceMethod(aClass, aSEL, newIMP, cd);
|
||||
```
|
||||
|
||||
- isa swizzling
|
||||
|
||||
```objective-c
|
||||
/// Represents an instance of a class.
|
||||
struct objc_object {
|
||||
Class _Nonnull isa OBJC_ISA_AVAILABILITY;
|
||||
};
|
||||
|
||||
/// A pointer to an instance of a class.
|
||||
typedef struct objc_object *id;
|
||||
|
||||
```
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -2556,7 +2841,7 @@ Method Swizzling 方法替换需要知道类名，我们在开发阶段使用 NS
|
||||
|
||||
|
||||
|
||||
#### 2.4 监控 App 常见网络请求
|
||||
#### 2.4 监控 App 常见网络请求<a name="categoryNameRules"></a>
|
||||
|
||||
本着成本的原因，由于现在大多数的项目的网络能力都是通过 [AFNetworking](https://github.com/AFNetworking/AFNetworking) 完成的，所以本文的网络监控可以快速完成。
|
||||
|
||||
|
||||
@@ -407,10 +407,10 @@ if ([cls respondsToSelector:sel]) {
|
||||
6. 网络请求完成后，通过 NetworkProtocolClient 将请求结果返回给 WKWebView。
|
||||
|
||||
|
||||
### 5. 拦截 WebView 内 Ajax 请求
|
||||
|
||||
|
||||
|
||||
|
||||
其实上述的方法也是可行，不过使用私有 API 的方式不是很推荐，一般在穷途末路的时候才选择私有 API，所以另一种思路是 hook Web 端的 ajax 请求。在执行 hook 后的 ajax 请求的时候将 ajax 的请求相关信息（请求方式、header、body 等）以 messageHandler 的方式告诉 Native，然后起到监控的效果。
|
||||
参考： https://www.jianshu.com/p/7337ac624b8e；https://github.com/wendux/Ajax-hook
|
||||
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user